Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…

… hack5_32
PaddlePaddle · Dec 12, 2023 · 59218c2 · 59218c2
2 parents 7ef61f8 + ec5e6b0
commit 59218c2
Show file tree

Hide file tree

Showing 2,896 changed files with 75,845 additions and 26,214 deletions.
diff --git a/.flake8 b/.flake8
@@ -1,5 +1,5 @@
 [flake8]
-select = C,E,W
+select = E
 exclude =
     ./build,
     # Exclude third-party libraries
@@ -19,12 +19,10 @@ ignore =
     # Do not assign a lambda expression, use a def
     E731,
     # Do not use variables named ‘l’, ‘O’, or ‘I’
-    E741,
-    # Line break before binary operator, it is not compatible with black
-    W503
+    E741
 per-file-ignores =
     # These files need tabs for testing.
-    test/dygraph_to_static/test_legacy_error.py:E101,W191
+    test/dygraph_to_static/test_legacy_error.py:E101
 
     # Ignore compare with True in sot unittest
     test/sot/test_dup_top.py:E712
diff --git a/.gitignore b/.gitignore
@@ -82,7 +82,7 @@ paddle/fluid/operators/generated_op*.cc
 paddle/fluid/operators/generated_sparse_op.cc
 paddle/fluid/operators/generated_static_op.cc
 paddle/fluid/operators/generated_fused_op.cc
-paddle/phi/ops/compat/generated_*.cc
+paddle/fluid/operators/ops_signature/generated_*.cc
 paddle/phi/api/yaml/parsed_apis/
 paddle/fluid/operators/generator/parsed_ops/
 paddle/fluid/pybind/tmp_eager_op_function_impl.h
@@ -95,9 +95,11 @@ paddle/fluid/framework/__init__.py
 paddle/phi/api/profiler/__init__.py
 python/paddle/incubate/fleet/parameter_server/pslib/ps_pb2.py
 paddle/phi/kernels/fusion/cutlass/conv2d/generated/*
+paddle/phi/kernels/fusion/cutlass/conv2d/generated_tmp/*
 python/paddle/base/incubate/fleet/parameter_server/pslib/ps_pb2.py
 paddle/fluid/ir_adaptor/translator/op_compat_info.cc
 paddle/phi/kernels/fusion/cutlass/cutlass_kernels/fpA_intB_gemm/autogen/*
+paddle/phi/kernels/fusion/cutlass/cutlass_kernels/fpA_intB_gemm/autogen_tmp/*
 paddle/fluid/pybind/static_op_function.*
 paddle/fluid/pybind/ops_api.cc
 paddle/fluid/pir/dialect/operator/ir/pd_api.*

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -60,7 +60,7 @@ option(WITH_IPU "Compile PaddlePaddle with Graphcore IPU" OFF)
 option(WITH_ONNXRUNTIME "Compile PaddlePaddle with ONNXRUNTIME" OFF)
 option(WITH_CUSPARSELT "Compile PaddlePaddle with CUSPARSELT" OFF)
 option(WITH_SETUP_INSTALL "Compile PaddlePaddle with setup.py" OFF)
-option(WITH_SHARED_PHI "Compile PaddlePaddle with SHARED LIB of PHI" OFF)
+option(WITH_SHARED_PHI "Compile PaddlePaddle with SHARED LIB of PHI" ON)
 option(CINN_ONLY "Compile CINN only in Paddle" OFF)
 option(CINN_WITH_CUDNN "Compile CINN with CUDNN support" ON)
 
@@ -124,6 +124,9 @@ if(WIN32)
   message("Build static library of PHI")
   set(CMAKE_SUPPRESS_REGENERATION ON)
   set(CMAKE_STATIC_LIBRARY_PREFIX lib)
+  set(WITH_SHARED_PHI
+      OFF
+      CACHE BOOL "Disable WITH_SHARED_PHI when compiling PADDLE ON WIN32" FORCE)
 
   set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} /bigobj")
   set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} /bigobj")
@@ -335,7 +338,7 @@ unset(WITH_RECORD_BUILDTIME CACHE)
 
 # PY_VERSION
 if(NOT PY_VERSION)
-  set(PY_VERSION 3.7)
+  set(PY_VERSION 3.8)
 endif()
 set(PYBIND11_PYTHON_VERSION ${PY_VERSION})
 
@@ -586,13 +589,6 @@ include(flags) # set paddle compile flags
 
 if(WITH_CINN)
   message(STATUS "Compile Paddle with CINN.")
-  message(
-    WARNING
-      "Enable WITH_SHARED_PHI when compiling with CINN. Force WITH_SHARED_PHI=ON."
-  )
-  set(WITH_SHARED_PHI
-      ON
-      CACHE BOOL "Enable WITH_SHARED_PHI when compiling with CINN" FORCE)
   # TODO(6clc): Use CINN_WITH_CUDNN to completely replace WITH_CUDNN in CINN.
   #             Use WITH_GPU to completely replace WITH_CUDA in CINN.
   set(WITH_MKL_CBLAS ${WITH_MKL})

diff --git a/README.md b/README.md
@@ -81,11 +81,20 @@ We provide [English](https://www.paddlepaddle.org.cn/documentation/docs/en/guide
 
    We appreciate your contributions!
 
-## Communication
+## Open Source Community
 
 - [Github Issues](https://github.com/PaddlePaddle/Paddle/issues): bug reports, feature requests, install issues, usage issues, etc.
-- QQ discussion group: 441226485 (PaddlePaddle).
-- [Forums](https://aistudio.baidu.com/paddle/forum): discuss implementations, research, etc.
+
+- Open Source Contribution Activities:
+
+  - Beginner: Happy Open Source Activity（[Regular Season](https://github.com/PaddlePaddle/Paddle/issues/56689)、[Pre-Hackathon Camp](https://github.com/PaddlePaddle/Paddle/issues/58497)）
+  - Advanced: PaddlePaddle Hackathon（[Personal Challenge Competition](https://github.com/PaddlePaddle/Paddle/issues/57262)、[LLM Application Competition](https://github.com/PaddlePaddle/Paddle/issues/57585)、[Hackathon Code Camp](https://github.com/PaddlePaddle/Paddle/issues/57264)）
+
+- Community Organizations:
+  - Technical Organization: [Paddle Framework Contributor Club, PFCC](https://github.com/PaddlePaddle/community/tree/master/pfcc)
+  - Community Governance Organization: [PaddlePaddle OpenSource Development Working Group, PPOSDWG](https://github.com/PaddlePaddle/community/tree/master/pposdwg)
+
+- Community Blog: https://pfcc.blog/
 
 ## Courses
 

diff --git a/README_cn.md b/README_cn.md
@@ -59,28 +59,35 @@ PaddlePaddle用户可领取**免费Tesla V100在线算力资源**，训练模型
 
 ## 文档
 
-我们提供 [英文](https://www.paddlepaddle.org.cn/documentation/docs/en/guides/index_en.html) 和
-[中文](https://www.paddlepaddle.org.cn/documentation/docs/zh/guides/index_cn.html) 文档
+我们提供 [英文](https://www.paddlepaddle.org.cn/documentation/docs/en/guides/index_en.html) 和 [中文](https://www.paddlepaddle.org.cn/documentation/docs/zh/guides/index_cn.html) 文档
 
-- [使用指南](https://www.paddlepaddle.org.cn/documentation/docs/zh/guides/index_cn.html)：或许您想从深度学习基础开始学习飞桨
+- [使用指南](https://www.paddlepaddle.org.cn/documentation/docs/zh/guides/index_cn.html)：或许你想从深度学习基础开始学习飞桨
 
-- [应用实践](https://www.paddlepaddle.org.cn/documentation/docs/zh/tutorial/index_cn.html)：使用飞桨搭建您的模型，更高效的完成深度学习任务
+- [应用实践](https://www.paddlepaddle.org.cn/documentation/docs/zh/tutorial/index_cn.html)：使用飞桨搭建你的模型，更高效的完成深度学习任务
 
 - [API 文档](https://www.paddlepaddle.org.cn/documentation/docs/zh/api/index_cn.html)：新的 API 支持代码更少更简洁的程序
 
 
-- [贡献方式](https://www.paddlepaddle.org.cn/documentation/docs/zh/guides/08_contribution/index_cn.html)：欢迎您的贡献!
+- [贡献方式](https://www.paddlepaddle.org.cn/documentation/docs/zh/guides/08_contribution/index_cn.html)：参与飞桨社区开源贡献的指南
 
-## 交流与反馈
+## 开源社区
 
-- 欢迎您通过[Github Issues](https://github.com/PaddlePaddle/Paddle/issues)来提交问题、报告与建议
-- QQ群: 441226485 (PaddlePaddle)
-- [论坛](https://aistudio.baidu.com/paddle/forum): 欢迎大家在PaddlePaddle论坛分享在使用PaddlePaddle中遇到的问题和经验，营造良好的论坛氛围
+- [Github Issues](https://github.com/PaddlePaddle/Paddle/issues)：提交安装/使用问题、报告bug、建议新特性、沟通开发计划等
+- 社区活动：
+
+  - 入门：快乐开源活动（[热身打卡 + 常规赛](https://github.com/PaddlePaddle/Paddle/issues/56689)、[启航计划](https://github.com/PaddlePaddle/Paddle/issues/58497)）
+  - 进阶：飞桨黑客马拉松（[开源贡献个人挑战赛](https://github.com/PaddlePaddle/Paddle/issues/57262)、[大模型应用与创意赛](https://github.com/PaddlePaddle/Paddle/issues/57585)、[飞桨护航计划集训营](https://github.com/PaddlePaddle/Paddle/issues/57264)）
+
+- 社区组织：
+  - 技术交流组织：[飞桨核心框架贡献者俱乐部 PFCC](https://github.com/PaddlePaddle/community/tree/master/pfcc)
+  - 社区治理组织：[飞桨社区开源发展工作组 PPOSDWG](https://github.com/PaddlePaddle/community/tree/master/pposdwg)
+
+- 社区博客：https://pfcc.blog/
 
 ## 课程
 
-- [服务器部署](https://aistudio.baidu.com/aistudio/course/introduce/19084): 详细介绍高性能服务器端部署实操，包含本地端及服务化Serving部署等
-- [端侧部署](https://aistudio.baidu.com/aistudio/course/introduce/22690): 详细介绍端侧多场景部署实操，从移动端设备、IoT、网页到小程序部署
+- [服务器部署](https://aistudio.baidu.com/aistudio/course/introduce/19084)：详细介绍高性能服务器端部署实操，包含本地端及服务化Serving部署等
+- [端侧部署](https://aistudio.baidu.com/aistudio/course/introduce/22690)：详细介绍端侧多场景部署实操，从移动端设备、IoT、网页到小程序部署
 
 ## 版权和许可证
 PaddlePaddle由[Apache-2.0 license](LICENSE)提供
diff --git a/cmake/cinn.cmake b/cmake/cinn.cmake
@@ -168,8 +168,8 @@ cinn_cc_library(
 add_dependencies(cinnapi GEN_LLVM_RUNTIME_IR_HEADER ZLIB::ZLIB)
 add_dependencies(cinnapi GEN_LLVM_RUNTIME_IR_HEADER ${core_deps})
 if(NOT CINN_ONLY)
-  target_link_libraries(cinnapi op_dialect phi)
-  add_dependencies(cinnapi op_dialect phi)
+  target_link_libraries(cinnapi op_dialect pir phi)
+  add_dependencies(cinnapi op_dialect pir phi)
 endif()
 
 target_link_libraries(cinnapi ${PYTHON_LIBRARIES})
@@ -231,8 +231,8 @@ function(gen_cinncore LINKTYPE)
   add_dependencies(${CINNCORE_TARGET} GEN_LLVM_RUNTIME_IR_HEADER ZLIB::ZLIB)
   add_dependencies(${CINNCORE_TARGET} GEN_LLVM_RUNTIME_IR_HEADER ${core_deps})
   if(NOT CINN_ONLY)
-    target_link_libraries(${CINNCORE_TARGET} op_dialect phi)
-    add_dependencies(${CINNCORE_TARGET} op_dialect phi)
+    target_link_libraries(${CINNCORE_TARGET} op_dialect pir phi)
+    add_dependencies(${CINNCORE_TARGET} op_dialect pir phi)
   endif()
 
   add_dependencies(${CINNCORE_TARGET} pybind)

diff --git a/cmake/external/xpu.cmake b/cmake/external/xpu.cmake
@@ -24,7 +24,7 @@ set(XPU_XFT_LIB_NAME "libxft.so")
 set(XPU_XPTI_LIB_NAME "libxpti.so")
 
 if(NOT DEFINED XPU_BASE_DATE)
-  set(XPU_BASE_DATE "20231128")
+  set(XPU_BASE_DATE "20231203")
 endif()
 set(XPU_XCCL_BASE_VERSION "1.1.6.1")
 if(NOT DEFINED XPU_XFT_BASE_VERSION)

diff --git a/cmake/generic.cmake b/cmake/generic.cmake
@@ -595,8 +595,8 @@ function(paddle_test_build TARGET_NAME)
     add_executable(${TARGET_NAME} ${paddle_test_SRCS})
     get_property(paddle_lib GLOBAL PROPERTY PADDLE_LIB_NAME)
     target_link_libraries(${TARGET_NAME} $<TARGET_LINKER_FILE:${paddle_lib}>
-                          ${paddle_test_DEPS} paddle_gtest_main_new)
-    add_dependencies(${TARGET_NAME} ${paddle_lib} ${paddle_test_DEPS}
+                          ${paddle_test_DEPS} common paddle_gtest_main_new)
+    add_dependencies(${TARGET_NAME} ${paddle_lib} ${paddle_test_DEPS} common
                      paddle_gtest_main_new)
     if(WITH_SHARED_PHI)
       target_link_libraries(${TARGET_NAME} $<TARGET_LINKER_FILE:phi>)
@@ -622,7 +622,7 @@ function(paddle_test_build TARGET_NAME)
     if(APPLE)
       target_link_libraries(
         ${TARGET_NAME}
-        "-Wl,-rpath,$<TARGET_FILE_DIR:${paddle_lib}> -Wl,-rpath,$<TARGET_FILE_DIR:phi> -Wl,-rpath,$<TARGET_FILE_DIR:pir>"
+        "-Wl,-rpath,$<TARGET_FILE_DIR:${paddle_lib}> -Wl,-rpath,$<TARGET_FILE_DIR:phi> -Wl,-rpath,$<TARGET_FILE_DIR:pir> -Wl,-rpath,$<TARGET_FILE_DIR:common>"
       )
     endif()
     common_link(${TARGET_NAME})

diff --git a/cmake/inference_lib.cmake b/cmake/inference_lib.cmake
@@ -286,6 +286,10 @@ copy(
 include_directories(${CMAKE_BINARY_DIR}/../paddle/fluid/framework/io)
 
 # copy api headers for phi & custom op
+copy(
+  inference_lib_dist
+  SRCS ${PADDLE_SOURCE_DIR}/paddle/common/*.h
+  DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/common/)
 copy(
   inference_lib_dist
   SRCS ${PADDLE_SOURCE_DIR}/paddle/phi/api/ext/*.h
@@ -304,8 +308,17 @@ copy(
   DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/phi/common/)
 copy(
   inference_lib_dist
-  SRCS ${PADDLE_SOURCE_DIR}/paddle/phi/core/macros.h
+  SRCS ${PADDLE_SOURCE_DIR}/paddle/phi/core/enforce.h
   DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/phi/core/)
+copy(
+  inference_lib_dist
+  SRCS ${PADDLE_SOURCE_DIR}/paddle/utils/string/*.h
+  DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/utils/string/)
+copy(
+  inference_lib_dist
+  SRCS ${PADDLE_SOURCE_DIR}/paddle/utils/string/tinyformat/tinyformat.h
+  DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/utils/string/tinyformat/
+)
 copy(
   inference_lib_dist
   SRCS ${PADDLE_SOURCE_DIR}/paddle/phi/core/visit_type.h
@@ -320,40 +333,13 @@ copy(
   DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/phi/)
 copy(
   inference_lib_dist
-  SRCS ${PADDLE_SOURCE_DIR}/paddle/utils/any.h
-  DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/utils/)
-copy(
-  inference_lib_dist
-  SRCS ${PADDLE_SOURCE_DIR}/paddle/utils/optional.h
-  DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/utils/)
-copy(
-  inference_lib_dist
-  SRCS ${PADDLE_SOURCE_DIR}/paddle/utils/none.h
-  DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/utils/)
-copy(
-  inference_lib_dist
-  SRCS ${PADDLE_SOURCE_DIR}/paddle/utils/flat_hash_map.h
-  DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/utils/)
-copy(
-  inference_lib_dist
-  SRCS ${PADDLE_SOURCE_DIR}/paddle/utils/flags.h
-  DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/utils/)
-copy(
-  inference_lib_dist
-  SRCS ${PADDLE_SOURCE_DIR}/paddle/utils/test_macros.h
+  SRCS ${PADDLE_SOURCE_DIR}/paddle/utils/*.h
   DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/utils/)
 copy(
   inference_lib_dist
   SRCS ${PADDLE_SOURCE_DIR}/paddle/extension.h
   DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/)
 
-if(NOT WITH_GFLAGS)
-  copy(
-    inference_lib_dist
-    SRCS ${PADDLE_SOURCE_DIR}/paddle/utils/flags_native.h
-    DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/utils/)
-endif()
-
 # the include path of phi needs to be changed to adapt to inference api path
 add_custom_command(
   TARGET inference_lib_dist

diff --git a/cmake/operators.cmake b/cmake/operators.cmake
@@ -688,7 +688,7 @@ function(prune_pybind_h)
 
   # add fused_op in op_list
   list(APPEND op_list "fc")
-  list(APPEND op_list "conv2d_fusion")
+  list(APPEND op_list "fused_conv2d_add_act")
   list(APPEND op_list "fusion_seqconv_eltadd_relu")
   list(APPEND op_list "fusion_seqpool_cvm_concat")
   list(APPEND op_list "fusion_gru")
@@ -734,3 +734,53 @@ function(prune_pybind_h)
     endif()
   endforeach()
 endfunction()
+
+function(append_op_util_declare TARGET)
+  file(READ ${TARGET} target_content)
+  string(REGEX MATCH "(PD_REGISTER_ARG_MAPPING_FN)\\([ \t\r\n]*[a-z0-9_]*"
+               util_registrar "${target_content}")
+  if(NOT ${util_registrar} EQUAL "")
+    string(REPLACE "PD_REGISTER_ARG_MAPPING_FN" "PD_DECLARE_ARG_MAPPING_FN"
+                   util_declare "${util_registrar}")
+    string(APPEND util_declare ");\n")
+    file(APPEND ${op_utils_header} "${util_declare}")
+  endif()
+endfunction()
+
+function(append_op_kernel_map_declare TARGET)
+  file(READ ${TARGET} target_content)
+  string(
+    REGEX
+      MATCH
+      "(PD_REGISTER_BASE_KERNEL_NAME)\\([ \t\r\n]*[a-z0-9_]*,[ \\\t\r\n]*[a-z0-9_]*"
+      kernel_mapping_registrar
+      "${target_content}")
+  if(NOT ${kernel_mapping_registrar} EQUAL "")
+    string(REPLACE "PD_REGISTER_BASE_KERNEL_NAME" "PD_DECLARE_BASE_KERNEL_NAME"
+                   kernel_mapping_declare "${kernel_mapping_registrar}")
+    string(APPEND kernel_mapping_declare ");\n")
+    file(APPEND ${op_utils_header} "${kernel_mapping_declare}")
+  endif()
+endfunction()
+
+function(register_op_utils TARGET_NAME)
+  set(utils_srcs)
+  set(options "")
+  set(oneValueArgs "")
+  set(multiValueArgs EXCLUDES DEPS)
+  cmake_parse_arguments(register_op_utils "${options}" "${oneValueArgs}"
+                        "${multiValueArgs}" ${ARGN})
+
+  file(GLOB SIGNATURES
+       "${PADDLE_SOURCE_DIR}/paddle/fluid/operators/ops_signature/*_sig.cc")
+  foreach(target ${SIGNATURES})
+    append_op_util_declare(${target})
+    append_op_kernel_map_declare(${target})
+    list(APPEND utils_srcs ${target})
+  endforeach()
+
+  cc_library(
+    ${TARGET_NAME}
+    SRCS ${utils_srcs}
+    DEPS ${register_op_utils_DEPS})
+endfunction()
diff --git a/cmake/phi.cmake b/cmake/phi.cmake
@@ -172,55 +172,6 @@ function(kernel_declare TARGET_LIST)
   endforeach()
 endfunction()
 
-function(append_op_util_declare TARGET)
-  file(READ ${TARGET} target_content)
-  string(REGEX MATCH "(PD_REGISTER_ARG_MAPPING_FN)\\([ \t\r\n]*[a-z0-9_]*"
-               util_registrar "${target_content}")
-  if(NOT ${util_registrar} EQUAL "")
-    string(REPLACE "PD_REGISTER_ARG_MAPPING_FN" "PD_DECLARE_ARG_MAPPING_FN"
-                   util_declare "${util_registrar}")
-    string(APPEND util_declare ");\n")
-    file(APPEND ${op_utils_header} "${util_declare}")
-  endif()
-endfunction()
-
-function(append_op_kernel_map_declare TARGET)
-  file(READ ${TARGET} target_content)
-  string(
-    REGEX
-      MATCH
-      "(PD_REGISTER_BASE_KERNEL_NAME)\\([ \t\r\n]*[a-z0-9_]*,[ \\\t\r\n]*[a-z0-9_]*"
-      kernel_mapping_registrar
-      "${target_content}")
-  if(NOT ${kernel_mapping_registrar} EQUAL "")
-    string(REPLACE "PD_REGISTER_BASE_KERNEL_NAME" "PD_DECLARE_BASE_KERNEL_NAME"
-                   kernel_mapping_declare "${kernel_mapping_registrar}")
-    string(APPEND kernel_mapping_declare ");\n")
-    file(APPEND ${op_utils_header} "${kernel_mapping_declare}")
-  endif()
-endfunction()
-
-function(register_op_utils TARGET_NAME)
-  set(utils_srcs)
-  set(options "")
-  set(oneValueArgs "")
-  set(multiValueArgs EXCLUDES DEPS)
-  cmake_parse_arguments(register_op_utils "${options}" "${oneValueArgs}"
-                        "${multiValueArgs}" ${ARGN})
-
-  file(GLOB SIGNATURES "${PADDLE_SOURCE_DIR}/paddle/phi/ops/compat/*_sig.cc")
-  foreach(target ${SIGNATURES})
-    append_op_util_declare(${target})
-    append_op_kernel_map_declare(${target})
-    list(APPEND utils_srcs ${target})
-  endforeach()
-
-  cc_library(
-    ${TARGET_NAME}
-    SRCS ${utils_srcs}
-    DEPS ${register_op_utils_DEPS})
-endfunction()
-
 function(prune_declaration_h)
   set(kernel_list ${KERNEL_LIST})
   file(STRINGS ${kernel_declare_file} kernel_registry_list)