diff --git a/cmake/configure.cmake b/cmake/configure.cmake
index 2c769a83ef496..35e78b01b9bbe 100644
--- a/cmake/configure.cmake
+++ b/cmake/configure.cmake
@@ -179,6 +179,9 @@ elseif(WITH_MUSA)
   add_definitions(-DPADDLE_WITH_MUSA)
   add_definitions(-DEIGEN_USE_GPU)
   add_definitions(-DEIGEN_USE_MUSA)
+  if(NOT MUDNN_FOUND)
+    message(FATAL_ERROR "Paddle needs MUDNN to compile")
+  endif()
 else()
   add_definitions(-DHPPL_STUB_FUNC)
   list(APPEND CMAKE_CXX_SOURCE_FILE_EXTENSIONS cu)
diff --git a/cmake/flags.cmake b/cmake/flags.cmake
index ba8f297ca0704..a32dea08e5bff 100644
--- a/cmake/flags.cmake
+++ b/cmake/flags.cmake
@@ -137,13 +137,12 @@ endif()
 # Do not care if this flag is support for gcc.
 
 # https://github.com/PaddlePaddle/Paddle/issues/12773
-# TODO(@caizhi): enable -Werror 
 if(NOT WIN32)
   set(COMMON_FLAGS
       -fPIC
       -fno-omit-frame-pointer
-      -Wall
       -Werror
+      -Wall
       -Wextra
       -Wno-unused-parameter
       -Wno-unused-function
diff --git a/cmake/generic.cmake b/cmake/generic.cmake
index b423ae0a4bb1b..b463a3377df56 100644
--- a/cmake/generic.cmake
+++ b/cmake/generic.cmake
@@ -787,7 +787,6 @@ function(musa_library TARGET_NAME)
     cmake_parse_arguments(musa_library "${options}" "${oneValueArgs}"
                           "${multiValueArgs}" ${ARGN})
     if(musa_library_SRCS)
-      # TODO(MTAI): enable compiling static library
       if(musa_library_SHARED OR musa_library_shared) # build *.so
         musa_add_library(${TARGET_NAME} SHARED ${musa_library_SRCS})
       else()
@@ -839,9 +838,25 @@ function(musa_binary TARGET_NAME)
   endif()
 endfunction()
 
-# TODO(@caizhi): enable musa_test
-#function(musa_test TARGET_NAME)
-#endfunction()
+function(musa_test TARGET_NAME)
+  if(WITH_MUSA AND WITH_TESTING)
+    set(oneValueArgs "")
+    set(multiValueArgs SRCS DEPS)
+    cmake_parse_arguments(musa_test "${options}" "${oneValueArgs}"
+                          "${multiValueArgs}" ${ARGN})
+    add_executable(${TARGET_NAME} ${musa_test_SRCS})
+    get_property(os_dependency_modules GLOBAL PROPERTY OS_DEPENDENCY_MODULES)
+    target_link_libraries(${TARGET_NAME} ${musa_test_DEPS}
+                          ${os_dependency_modules} paddle_gtest_main phi)
+    add_dependencies(${TARGET_NAME} ${musa_test_DEPS} paddle_gtest_main)
+    common_link(${TARGET_NAME})
+    add_test(${TARGET_NAME} ${TARGET_NAME})
+    set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT
+                                              FLAGS_cpu_deterministic=true)
+    set_property(TEST ${TARGET_NAME} PROPERTY ENVIRONMENT
+                                              FLAGS_init_allocated_mem=true)
+  endif()
+endfunction()
 
 function(xpu_library TARGET_NAME)
   if(WITH_XPU_KP)
@@ -1342,6 +1357,11 @@ function(math_library TARGET)
       ${TARGET}
       SRCS ${cc_srcs} ${cu_srcs}
       DEPS ${math_library_DEPS} ${math_common_deps})
+  elseif(WITH_MUSA)
+    musa_library(
+      ${TARGET}
+      SRCS ${cc_srcs} ${cu_srcs}
+      DEPS ${math_library_DEPS} ${math_common_deps})
   elseif(${cc_srcs_len} GREATER 0)
     cc_library(
       ${TARGET}
diff --git a/cmake/mccl.cmake b/cmake/mccl.cmake
index 12191a2711d46..6d39b39434e62 100644
--- a/cmake/mccl.cmake
+++ b/cmake/mccl.cmake
@@ -10,7 +10,7 @@ endif()
 # FIXME(MTAI): please make sure that we can find MCCL successfully
 if(WITH_MCCL)
   set(MCCL_ROOT
-      ${MUSA_PATH}/mccl
+      "/usr/local/musa/"
       CACHE PATH "MCCL ROOT")
   find_path(
     MCCL_INCLUDE_DIR mccl.h
@@ -18,14 +18,35 @@ if(WITH_MCCL)
           $ENV{MCCL_ROOT} $ENV{MCCL_ROOT}/include $ENV{MCCL_ROOT}/local/include
     NO_DEFAULT_PATH)
 
-  file(READ ${MCCL_INCLUDE_DIR}/mccl.h MCCL_VERSION_FILE_CONTENTS)
+  if(MCCL_INCLUDE_DIR)
+    file(READ ${MCCL_INCLUDE_DIR}/mccl.h MCCL_VERSION_FILE_CONTENTS)
 
-  string(REGEX MATCH "define NCCL_VERSION_CODE +([0-9]+)" MCCL_VERSION
-               "${MCCL_VERSION_FILE_CONTENTS}")
-  string(REGEX REPLACE "define NCCL_VERSION_CODE +([0-9]+)" "\\1" MCCL_VERSION
-                       "${MCCL_VERSION}")
+    string(REGEX MATCH "define MCCL_MAJOR +([0-9]+)" MCCL_MAJOR_VERSION
+                 "${MCCL_VERSION_FILE_CONTENTS}")
+    string(REGEX REPLACE "define MCCL_MAJOR +([0-9]+)" "\\1" MCCL_MAJOR_VERSION
+                 "${MCCL_MAJOR_VERSION}")
+    string(REGEX MATCH "define MCCL_MINOR +([0-9]+)" MCCL_MINOR_VERSION
+                 "${MCCL_VERSION_FILE_CONTENTS}")
+    string(REGEX REPLACE "define MCCL_MINOR +([0-9]+)" "\\1" MCCL_MINOR_VERSION
+    		 "${MCCL_MINOR_VERSION}")
+    string(REGEX MATCH "define MCCL_PATCH +([0-9]+)" MCCL_PATCH_VERSION
+                 "${MCCL_VERSION_FILE_CONTENTS}")
+    string(REGEX REPLACE "define MCCL_PATCH +([0-9]+)" "\\1" MCCL_PATCH_VERSION
+                 "${MCCL_PATCH_VERSION}")
+    if(NOT MCCL_MAJOR_VERSION)
+      set(MCCL_VERSION "???")
+    else()
+      math(EXPR MCCL_VERSION "${MCCL_MAJOR_VERSION} * 1000 +
+                 ${MCCL_MINOR_VERSION} * 100 + ${MCCL_PATCH_VERSION}")
+    endif()
+    add_definitions("-DMCCL_VERSION_CODE=$MCCL_VERSION")
+    include_directories(${MCCL_INCLUDE_DIR})
 
-  message(STATUS "Current MCCL header is ${MCCL_INCLUDE_DIR}/mccl.h. "
-                 "Current MCCL version is v${MCCL_VERSION}. ")
+    message(STATUS "Current MCCL header is ${MCCL_INCLUDE_DIR}/mccl.h. ")
+    message(STATUS "Current MCCL version is "
+        "v${MCCL_MAJOR_VERSION}.${MCCL_MINOR_VERSION}.${MCCL_PATCH_VERSION} ")
+  else()
+	  message(FATAL_ERROR "WITH_MCCL is enabled but mccl.h file is not found!")
+  endif()
 endif()
 
diff --git a/cmake/mudnn.cmake b/cmake/mudnn.cmake
index 4485139f7d01c..ab66620ad4c26 100644
--- a/cmake/mudnn.cmake
+++ b/cmake/mudnn.cmake
@@ -4,6 +4,10 @@ endif()
 
 if(WIN32)
   return()
+else()
+  set(MUDNN_ROOT
+      "/usr/local/musa"
+      CACHE PATH "MUDNN ROOT")
 endif()
 
 find_path(
@@ -12,8 +16,6 @@ find_path(
         $ENV{MUDNN_ROOT}/include ${MUSA_TOOLKIT_INCLUDE}
   NO_DEFAULT_PATH)
 
-get_filename_component(__libpath_hist ${MUSA_MUSART_LIBRARY} PATH)
-
 set(TARGET_ARCH "x86_64")
 if(NOT ${CMAKE_SYSTEM_PROCESSOR})
   set(TARGET_ARCH ${CMAKE_SYSTEM_PROCESSOR})
@@ -27,7 +29,6 @@ list(
   ${MUDNN_ROOT}/lib
   ${MUDNN_ROOT}/lib/x64
   ${MUDNN_ROOT}/lib/${TARGET_ARCH}-linux-gnu
-  ${MUDNN_ROOT}/local/cuda-${MUSA_VERSION}/targets/${TARGET_ARCH}-linux/lib/
   $ENV{MUDNN_ROOT}
   $ENV{MUDNN_ROOT}/lib64
   $ENV{MUDNN_ROOT}/lib
@@ -44,7 +45,7 @@ endif()
 find_library(
   MUDNN_LIBRARY
   NAMES ${MUDNN_LIB_NAME}
-  PATHS ${MUDNN_CHECK_LIBRARY_DIRS} ${MUDNN_INCLUDE_DIR} ${__libpath_hist}
+  PATHS ${MUDNN_CHECK_LIBRARY_DIRS} ${MUDNN_INCLUDE_DIR}
   NO_DEFAULT_PATH
   DOC "Path to muDNN library.")
 
@@ -54,14 +55,38 @@ else()
   set(MUDNN_FOUND OFF)
 endif()
 
-# TODO(@caizhi): enable mudnn finding
-#macro(find_cudnn_version cudnn_header_file)
-#endmacro()
+macro(find_mudnn_version mudnn_version_file)
+  file(READ ${mudnn_version_file} MUDNN_VERSION_FILE_CONTENTS)
+  get_filename_component(MUDNN_LIB_PATH ${MUDNN_LIBRARY} DIRECTORY)
 
-#if(MUDNN_FOUND)
-#  find_mudnn_version(${MUDNN_INCLUDE_DIR}/mudnn.h)
-#  if(NOT MUDNN_MAJOR_VERSION)
-#    find_mudnn_version(${MUDNN_INCLUDE_DIR}/mudnn_version.h)
-#  endif()
-#endif()
+  string(REGEX MATCH "define MUDNN_VERSION_MAJOR +([0-9]+)" MUDNN_MAJOR_VERSION
+               "${MUDNN_VERSION_FILE_CONTENTS}")
+  string(REGEX REPLACE "define MUDNN_VERSION_MAJOR +([0-9]+)" "\\1"
+               MUDNN_MAJOR_VERSION "${MUDNN_MAJOR_VERSION}")
+  string(REGEX MATCH "define MUDNN_VERSION_MINOR +([0-9]+)" MUDNN_MINOR_VERSION
+               "${MUDNN_VERSION_FILE_CONTENTS}")
+  string(REGEX REPLACE "define MUDNN_VERSION_MINOR +([0-9]+)" "\\1"
+	       MUDNN_MINOR_VERSION "${MUDNN_MINOR_VERSION}")
+  string(REGEX MATCH "define MUDNN_VERSION_PATCH +([0-9]+)" MUDNN_PATCH_VERSION
+               "${MUDNN_VERSION_FILE_CONTENTS}")
+  string(REGEX REPLACE "define MUDNN_VERSION_PATCH +([0-9]+)" "\\1"
+               MUDNN_PATCH_VERSION "${MUDNN_PATCH_VERSION}")
 
+  if(NOT MUDNN_MAJOR_VERSION)
+    set(MUDNN_VERSION "???")
+  else()
+    add_definitions("-DMUDNN_MAJOR_VERSION=\"${MUDNN_MAJOR_VERSION}\"")
+    math(EXPR MUDNN_VERSION "${MUDNN_MAJOR_VERSION} * 1000 +
+               ${MUDNN_MINOR_VERSION} * 100 + ${MUDNN_PATCH_VERSION}")
+    message(STATUS "Current muDNN version file is ${mudnn_version_file} ")
+    message(
+      STATUS
+        "Current muDNN version is v${MUDNN_MAJOR_VERSION}.${MUDNN_MINOR_VERSION}.${MUDNN_PATCH_VERSION}. "
+    )
+  endif()
+endmacro()
+
+if(MUDNN_FOUND)
+  find_mudnn_version(${MUDNN_INCLUDE_DIR}/mudnn_version.h)
+  include_directories(${MUDNN_INCLUDE_DIR})
+endif()
diff --git a/cmake/musa.cmake b/cmake/musa.cmake
index b6192ca60a26e..487db6e61ba7d 100644
--- a/cmake/musa.cmake
+++ b/cmake/musa.cmake
@@ -15,16 +15,73 @@ set(CMAKE_MODULE_PATH "${MUSA_PATH}/cmake" ${CMAKE_MODULE_PATH})
 
 find_package(MUSA REQUIRED)
 include_directories(${MUSA_PATH}/include)
-include_directories(/usr/lib/llvm-11/include/openmp/)
 
-# TODO(@caizhi): enable finding musa version
-#macro(find_musa_version version_file)
-#endmacro()
-#find_musa_version(${MUSA_PATH}/version.h)
+# set openmp include directory
+set(llvm_search_list)
+foreach(item RANGE 6 20 1)
+  list(APPEND llvm_search_list /usr/lib/llvm-${item}/include/openmp/)
+endforeach()
+
+find_path(
+  OPENMP_INCLUDE_DIR omp.h
+  PATHS ${llvm_search_list}
+  REQUIRED
+  NO_DEFAULT_PATH)
+include_directories(${OPENMP_INCLUDE_DIR})
+
+macro(find_musa_version musa_version_file)
+  set(python_file ${PROJECT_BINARY_DIR}/get_version.py)
+  set(MUSA_VERSION "None" CACHE STRING "musa version" FORCE)
+  file(
+    WRITE ${python_file}
+    ""
+    "import json\n"
+    "import sys\n"
+    "with open(sys.argv[1], 'r') as f:\n"
+    "    data = json.load(f)\n"
+    "    print(data[\"MUSA_RUNTIME\"][\"version\"])"
+    "")
+
+  execute_process(
+    COMMAND "python" "${python_file}" ${musa_version_file}
+    WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/CMakeFiles/"
+    RESULT_VARIABLE python_res
+    OUTPUT_VARIABLE python_out
+    ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+  if(python_res EQUAL 0)
+    set(MUSA_VERSION ${python_out})
+  endif()
+  string(REGEX REPLACE "([0-9]+)\.([0-9]+)\.([0-9]+)" "\\1" MUSA_MAJOR_VERSION "${MUSA_VERSION}")
+  string(REGEX REPLACE "([0-9]+)\.([0-9]+)\.([0-9]+)" "\\2" MUSA_MINOR_VERSION "${MUSA_VERSION}")
+  string(REGEX REPLACE "([0-9]+)\.([0-9]+)\.([0-9]+)" "\\3" MUSA_PATCH_VERSION "${MUSA_VERSION}")
+
+  if(NOT MUSA_MAJOR_VERSION)
+    set(MUSA_VERSION "???")
+    message(
+      WARNING "Cannot find MUSA version in ${MUSA_PATH}/version.json"
+    )
+  else()
+    math(
+      EXPR
+      MUSA_VERSION
+      "${MUSA_MAJOR_VERSION} * 10000 + ${MUSA_MINOR_VERSION} * 100   + ${MUSA_PATCH_VERSION}"
+    )
+    message(
+      STATUS
+        "Current MUSA version file is ${MUSA_PATH}/version.json.")
+    message(
+      STATUS
+	"Current MUSA version is v${MUSA_MAJOR_VERSION}.${MUSA_MINOR_VERSION}.${MUSA_PATCH_VERSION} ")
+  endif()
+endmacro()
+find_musa_version(${MUSA_PATH}/version.json)
 
 list(APPEND MUSA_MCC_FLAGS -Wno-unknown-warning-option)
 list(APPEND MUSA_MCC_FLAGS -Wno-macro-redefined)
 list(APPEND MUSA_MCC_FLAGS -Wno-unused-variable)
+list(APPEND MUSA_MCC_FLAGS -Wno-deprecated-copy-with-user-provided-copy)
+list(APPEND MUSA_MCC_FLAGS -Wno-pragma-once-outside-header)
 list(APPEND MUSA_MCC_FLAGS -Wno-return-type)
 list(APPEND MUSA_MCC_FLAGS -Wno-sign-compare)
 list(APPEND MUSA_MCC_FLAGS -Wno-mismatched-tags)
diff --git a/cmake/version.cmake b/cmake/version.cmake
index e6707665a3851..039ae58a64c00 100644
--- a/cmake/version.cmake
+++ b/cmake/version.cmake
@@ -86,12 +86,18 @@ function(version version_file)
     "WITH_MKLDNN: ${WITH_MKLDNN}\n"
     "WITH_GPU: ${WITH_GPU}\n"
     "WITH_ROCM: ${WITH_ROCM}\n"
+    "WITH_MUSA: ${WITH_MUSA}\n"
     "WITH_IPU: ${WITH_IPU}\n")
   if(WITH_GPU)
     file(APPEND ${version_file}
          "CUDA version: ${CUDA_VERSION}\n"
          "CUDNN version: v${CUDNN_MAJOR_VERSION}.${CUDNN_MINOR_VERSION}\n")
   endif()
+  if(WITH_MUSA)
+    file(APPEND ${version_file}
+         "MUSA version: v${MUSA_MAJOR_VERSION}.${MUSA_MINOR_VERSION}.${MUSA_PATCH_VERSION}\n"
+         "MUDNN version: v${MUDNN_MAJOR_VERSION}.${MUDNN_MINOR_VERSION}\n")
+  endif()
   if(WITH_ROCM)
     file(APPEND ${version_file}
          "HIP version: v${HIP_MAJOR_VERSION}.${HIP_MINOR_VERSION}\n"
diff --git a/paddle/fluid/operators/math/CMakeLists.txt b/paddle/fluid/operators/math/CMakeLists.txt
index af14333b9d1ea..7b14e8541fd02 100644
--- a/paddle/fluid/operators/math/CMakeLists.txt
+++ b/paddle/fluid/operators/math/CMakeLists.txt
@@ -21,5 +21,6 @@ endif()
 
 math_library(unpooling)
 math_library(prelu)
-math_library(bert_encoder_functor)
+# TODO(@caizhi): enable it
+#math_library(bert_encoder_functor)
 math_library(tree2col DEPS phi)
diff --git a/paddle/fluid/operators/math/sample_prob.cu b/paddle/fluid/operators/math/sample_prob.cu
index 53819488ac02b..5ab90409df1e7 100644
--- a/paddle/fluid/operators/math/sample_prob.cu
+++ b/paddle/fluid/operators/math/sample_prob.cu
@@ -164,7 +164,7 @@ void GPUSampleWithProb<T>::operator()(const phi::GPUContext& context,
   PADDLE_ENFORCE_GPU_SUCCESS(musaMemcpy(samples_data + num_true,
                                         s_data,
                                         sizeof(int64_t) * num_samples,
-                                        hipMemcpyHostToDevice));
+                                        musaMemcpyHostToDevice));
 #else
   PADDLE_ENFORCE_GPU_SUCCESS(cudaMemcpy(samples_data + num_true,
                                         s_data,
diff --git a/paddle/fluid/platform/device/gpu/gpu_launch_config.h b/paddle/fluid/platform/device/gpu/gpu_launch_config.h
index 8ce858b4d37a1..fb38b790c8540 100644
--- a/paddle/fluid/platform/device/gpu/gpu_launch_config.h
+++ b/paddle/fluid/platform/device/gpu/gpu_launch_config.h
@@ -20,6 +20,8 @@
 
 #ifdef PADDLE_WITH_CUDA
 #include <cuda_runtime.h>
+#elif defined(PADDLE_WITH_MUSA)
+#include <musa_runtime.h>
 #else
 #include <hip/hip_runtime.h>
 #endif