From 315179f73b24aab50de5e4efbd8db4b92543f59a Mon Sep 17 00:00:00 2001 From: risemeup1 <62429225+risemeup1@users.noreply.github.com> Date: Wed, 14 Dec 2022 10:29:06 +0800 Subject: [PATCH] Support ninja (#48932) * move inplace_apis_indygraph_only from paddle.flud.dygraph.inplace_utils to paddle.utils * modify conflict * modify conflict * modify conflict * modify conflict * modify conflict * modify conflict * modify conflict * modify static-check ci error * fix conflict * modify failed tests * fix conflict * fix conflict * fix pool2d examples * modify conflict * fix failed tests * fix conflict * fix failed tests * modfiy problem of deleting pool2d * support Ninja in setup.py * support different cmake_generators * modify after reviewed * delete unused denotes --- setup.py | 169 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 96 insertions(+), 73 deletions(-) diff --git a/setup.py b/setup.py index 6d088750a60b0..a1ad5e6ecd933 100644 --- a/setup.py +++ b/setup.py @@ -557,12 +557,21 @@ def options_process(args, build_options): args.append("-D{}={}".format(key, value)) +def get_cmake_generator(): + if os.getenv("CMAKE_GENERATOR"): + cmake_generator = os.getenv("CMAKE_GENERATOR") + else: + cmake_generator = "Unix Makefiles" + + return cmake_generator + + def cmake_run(args, build_path): with cd(build_path): cmake_args = [] cmake_args.append(CMAKE) - cmake_args.append('-DWITH_SETUP_INSTALL=ON') cmake_args += args + cmake_args.append('-DWITH_SETUP_INSTALL=ON') cmake_args.append(TOP_DIR) print("cmake_args:", cmake_args) subprocess.check_call(cmake_args) @@ -573,7 +582,6 @@ def build_run(args, build_path, envrion_var): build_args = [] build_args.append(CMAKE) build_args += args - # cmake_args.append(TOP_DIR) print(" ".join(build_args)) try: subprocess.check_call(build_args, cwd=build_path, env=envrion_var) @@ -581,6 +589,23 @@ def build_run(args, build_path, envrion_var): sys.exit(1) +def run_cmake_build(build_path): + build_args = ["--build", ".", "--target", "install", "--config", 'Release'] + max_jobs = os.getenv("MAX_JOBS") + if max_jobs is not None: + max_jobs = max_jobs or str(multiprocessing.cpu_count()) + + build_args += ["--"] + if IS_WINDOWS: + build_args += ["/p:CL_MPCount={}".format(max_jobs)] + else: + build_args += ["-j", max_jobs] + else: + build_args += ["-j", str(multiprocessing.cpu_count())] + environ_var = os.environ.copy() + build_run(build_args, build_path, environ_var) + + def build_steps(): print('------- Building start ------') build_dir = os.getenv("BUILD_DIR") @@ -596,83 +621,81 @@ def build_steps(): # if rerun_cmake is True,remove CMakeCache.txt and rerun camke if os.path.isfile(cmake_cache_file_path) and rerun_cmake is True: os.remove(cmake_cache_file_path) - if not os.path.exists(cmake_cache_file_path): - env_var = os.environ.copy() # get env variables - paddle_build_options = {} - other_options = {} - other_options.update( - { - option: option - for option in ( - "PYTHON_LIBRARY", - "INFERENCE_DEMO_INSTALL_DIR", - "ON_INFER", - "PYTHON_EXECUTABLE", - "TENSORRT_ROOT", - "CUDA_ARCH_NAME", - "CUDA_ARCH_BIN", - "PYTHON_INCLUDE_DIR", - "PYTHON_LIBRARIES", - "PY_VERSION", - "CUB_PATH", - "NEW_RELEASE_PYPI", - "CUDNN_ROOT", - "THIRD_PARTY_PATH", - "NOAVX_CORE_FILE", - "LITE_GIT_TAG", - "CUDA_TOOLKIT_ROOT_DIR", - "NEW_RELEASE_JIT", - "XPU_SDK_ROOT", - "MSVC_STATIC_CRT", - "Ninja", - "NEW_RELEASE_ALL", - ) - } - ) - # if environment variables which start with "WITH_" or "CMAKE_",put it into build_options - for option_key, option_value in env_var.items(): - if option_key.startswith(("CMAKE_", "WITH_")): - paddle_build_options[option_key] = option_value - if option_key in other_options: - print("type:", type(other_options[option_key])) - if ( - option_key == 'PYTHON_EXECUTABLE' - or option_key == 'PYTHON_LIBRARY' - or option_key == 'PYTHON_LIBRARIES' - ): - key = option_key + ":FILEPATH" - print(key) - elif option_key == 'PYTHON_INCLUDE_DIR': - key = key = option_key + ':PATH' - print(key) - else: - key = other_options[option_key] - if key not in paddle_build_options: - paddle_build_options[key] = option_value - args = [] - options_process(args, paddle_build_options) - print("args:", args) - cmake_run(args, build_path) + + CMAKE_GENERATOR = get_cmake_generator() + + if CMAKE_GENERATOR == "Ninja": + build_ninja_file_path = os.path.join(build_path, "build.ninja") + # if os.path.exists(cmake_cache_file_path) and not (USE_NINJA and not os.path.exists(build_ninja_file_path)): + if os.path.exists(cmake_cache_file_path) and os.path.exists( + build_ninja_file_path + ): + print("Do not need rerun camke,everything is ready,run build now") + run_cmake_build(build_path) + return + + args = [] + env_var = os.environ.copy() # get env variables + paddle_build_options = {} + other_options = {} + other_options.update( + { + option: option + for option in ( + "PYTHON_LIBRARY", + "INFERENCE_DEMO_INSTALL_DIR", + "ON_INFER", + "PYTHON_EXECUTABLE", + "TENSORRT_ROOT", + "CUDA_ARCH_NAME", + "CUDA_ARCH_BIN", + "PYTHON_INCLUDE_DIR", + "PYTHON_LIBRARIES", + "PY_VERSION", + "CUB_PATH", + "NEW_RELEASE_PYPI", + "CUDNN_ROOT", + "THIRD_PARTY_PATH", + "NOAVX_CORE_FILE", + "LITE_GIT_TAG", + "CUDA_TOOLKIT_ROOT_DIR", + "NEW_RELEASE_JIT", + "XPU_SDK_ROOT", + "MSVC_STATIC_CRT", + "NEW_RELEASE_ALL", + "CMAKE_GENERATOR", + ) + } + ) + # if environment variables which start with "WITH_" or "CMAKE_",put it into build_options + for option_key, option_value in env_var.items(): + if option_key.startswith(("CMAKE_", "WITH_")): + paddle_build_options[option_key] = option_value + if option_key in other_options: + if ( + option_key == 'PYTHON_EXECUTABLE' + or option_key == 'PYTHON_LIBRARY' + or option_key == 'PYTHON_LIBRARIES' + ): + key = option_key + ":FILEPATH" + print(key) + elif option_key == 'PYTHON_INCLUDE_DIR': + key = key = option_key + ':PATH' + print(key) + else: + key = other_options[option_key] + if key not in paddle_build_options: + paddle_build_options[key] = option_value + options_process(args, paddle_build_options) + print("args:", args) + cmake_run(args, build_path) # make if only_cmake: print( "You have finished running cmake, the program exited,run 'ccmake build' to adjust build options and 'python setup.py install to build'" ) sys.exit() - build_args = ["--build", ".", "--target", "install", "--config", 'Release'] - max_jobs = os.getenv("MAX_JOBS") - if max_jobs is not None: - max_jobs = max_jobs or str(multiprocessing.cpu_count()) - - build_args += ["--"] - if IS_WINDOWS: - build_args += ["/p:CL_MPCount={}".format(max_jobs)] - else: - build_args += ["-j", max_jobs] - else: - build_args += ["-j", str(multiprocessing.cpu_count())] - environ_var = os.environ.copy() - build_run(build_args, build_path, environ_var) + run_cmake_build(build_path) def get_setup_requires():