Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support ninja #48932

Merged
merged 28 commits into from
Dec 14, 2022
Merged
Changes from 26 commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
96c3107
move inplace_apis_indygraph_only from paddle.flud.dygraph.inplace_uti…
risemeup1 Dec 5, 2022
751241c
modify conflict
risemeup1 Dec 5, 2022
9ec77cb
modify conflict
risemeup1 Dec 5, 2022
654caca
modify conflict
risemeup1 Dec 5, 2022
f440ac8
modify conflict
risemeup1 Dec 5, 2022
a0a878c
modify conflict
risemeup1 Dec 5, 2022
3928713
Merge branch 'develop' into inplace_apis_in_dygraph_only
risemeup1 Dec 5, 2022
3ea1763
modify conflict
risemeup1 Dec 5, 2022
1f34455
modify conflict
risemeup1 Dec 5, 2022
e7ee1d9
modify static-check ci error
risemeup1 Dec 5, 2022
b70ca48
fix conflict
risemeup1 Dec 5, 2022
28327b9
modify failed tests
risemeup1 Dec 6, 2022
552b206
fix conflict
risemeup1 Dec 6, 2022
c037293
fix conflict
risemeup1 Dec 6, 2022
ac5e94f
fix pool2d examples
risemeup1 Dec 6, 2022
942933a
modify conflict
risemeup1 Dec 6, 2022
a9cc93c
fix failed tests
risemeup1 Dec 6, 2022
07af546
fix conflict
risemeup1 Dec 6, 2022
3b38465
fix failed tests
risemeup1 Dec 7, 2022
dcdb91f
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…
risemeup1 Dec 7, 2022
0b8e3cc
modfiy problem of deleting pool2d
risemeup1 Dec 7, 2022
b4f9202
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…
risemeup1 Dec 7, 2022
1e03f01
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…
risemeup1 Dec 8, 2022
c2d89c5
support Ninja in setup.py
risemeup1 Dec 8, 2022
b34a902
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into…
risemeup1 Dec 9, 2022
ba376ca
support different cmake_generators
risemeup1 Dec 9, 2022
7e81689
modify after reviewed
risemeup1 Dec 12, 2022
797b18e
delete unused denotes
risemeup1 Dec 12, 2022
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
167 changes: 95 additions & 72 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -557,12 +557,21 @@ def options_process(args, build_options):
args.append("-D{}={}".format(key, value))


def get_cmake_generators():
if os.getenv("CMAKE_GENERATOR"):
cmake_generators = os.getenv("CMAKE_GENERATOR")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cmake_generators -> cmake_generator

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok.thks

else:
cmake_generators = " Unix Makefiles"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
cmake_generators = " Unix Makefiles"
cmake_generators = "Unix Makefiles"

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok,thks


return cmake_generators


def cmake_run(args, build_path):
with cd(build_path):
cmake_args = []
cmake_args.append(CMAKE)
cmake_args.append('-DWITH_SETUP_INSTALL=ON')
cmake_args += args
cmake_args.append('-DWITH_SETUP_INSTALL=ON')
cmake_args.append(TOP_DIR)
print("cmake_args:", cmake_args)
subprocess.check_call(cmake_args)
Expand All @@ -581,6 +590,23 @@ def build_run(args, build_path, envrion_var):
sys.exit(1)


def run_cmake_build(build_path):
build_args = ["--build", ".", "--target", "install", "--config", 'Release']
max_jobs = os.getenv("MAX_JOBS")
if max_jobs is not None:
max_jobs = max_jobs or str(multiprocessing.cpu_count())

build_args += ["--"]
if IS_WINDOWS:
build_args += ["/p:CL_MPCount={}".format(max_jobs)]
else:
build_args += ["-j", max_jobs]
else:
build_args += ["-j", str(multiprocessing.cpu_count())]
environ_var = os.environ.copy()
build_run(build_args, build_path, environ_var)


def build_steps():
print('------- Building start ------')
build_dir = os.getenv("BUILD_DIR")
Expand All @@ -596,83 +622,80 @@ def build_steps():
# if rerun_cmake is True,remove CMakeCache.txt and rerun camke
if os.path.isfile(cmake_cache_file_path) and rerun_cmake is True:
os.remove(cmake_cache_file_path)
if not os.path.exists(cmake_cache_file_path):
env_var = os.environ.copy() # get env variables
paddle_build_options = {}
other_options = {}
other_options.update(
{
option: option
for option in (
"PYTHON_LIBRARY",
"INFERENCE_DEMO_INSTALL_DIR",
"ON_INFER",
"PYTHON_EXECUTABLE",
"TENSORRT_ROOT",
"CUDA_ARCH_NAME",
"CUDA_ARCH_BIN",
"PYTHON_INCLUDE_DIR",
"PYTHON_LIBRARIES",
"PY_VERSION",
"CUB_PATH",
"NEW_RELEASE_PYPI",
"CUDNN_ROOT",
"THIRD_PARTY_PATH",
"NOAVX_CORE_FILE",
"LITE_GIT_TAG",
"CUDA_TOOLKIT_ROOT_DIR",
"NEW_RELEASE_JIT",
"XPU_SDK_ROOT",
"MSVC_STATIC_CRT",
"Ninja",
"NEW_RELEASE_ALL",
)
}
)
# if environment variables which start with "WITH_" or "CMAKE_",put it into build_options
for option_key, option_value in env_var.items():
if option_key.startswith(("CMAKE_", "WITH_")):
paddle_build_options[option_key] = option_value
if option_key in other_options:
print("type:", type(other_options[option_key]))
if (
option_key == 'PYTHON_EXECUTABLE'
or option_key == 'PYTHON_LIBRARY'
or option_key == 'PYTHON_LIBRARIES'
):
key = option_key + ":FILEPATH"
print(key)
elif option_key == 'PYTHON_INCLUDE_DIR':
key = key = option_key + ':PATH'
print(key)
else:
key = other_options[option_key]
if key not in paddle_build_options:
paddle_build_options[key] = option_value
args = []
options_process(args, paddle_build_options)
print("args:", args)
cmake_run(args, build_path)

CMAKE_GENERATOR = get_cmake_generators()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

get_cmake_generators -> get_cmake_generator

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok,thks


if CMAKE_GENERATOR == "Ninja":
build_ninja_file_path = os.path.join(build_path, "build.ninja")
# if os.path.exists(cmake_cache_file_path) and not (USE_NINJA and not os.path.exists(build_ninja_file_path)):
if os.path.exists(cmake_cache_file_path) and os.path.exists(
build_ninja_file_path
):
print("Do not need rerun camke,everything is ready,run build now")
run_cmake_build(build_path)
return
args = []
env_var = os.environ.copy() # get env variables
paddle_build_options = {}
other_options = {}
other_options.update(
{
option: option
for option in (
"PYTHON_LIBRARY",
"INFERENCE_DEMO_INSTALL_DIR",
"ON_INFER",
"PYTHON_EXECUTABLE",
"TENSORRT_ROOT",
"CUDA_ARCH_NAME",
"CUDA_ARCH_BIN",
"PYTHON_INCLUDE_DIR",
"PYTHON_LIBRARIES",
"PY_VERSION",
"CUB_PATH",
"NEW_RELEASE_PYPI",
"CUDNN_ROOT",
"THIRD_PARTY_PATH",
"NOAVX_CORE_FILE",
"LITE_GIT_TAG",
"CUDA_TOOLKIT_ROOT_DIR",
"NEW_RELEASE_JIT",
"XPU_SDK_ROOT",
"MSVC_STATIC_CRT",
"NEW_RELEASE_ALL",
"CMAKE_GENERATOR",
)
}
)
# if environment variables which start with "WITH_" or "CMAKE_",put it into build_options
for option_key, option_value in env_var.items():
if option_key.startswith(("CMAKE_", "WITH_")):
paddle_build_options[option_key] = option_value
if option_key in other_options:
if (
option_key == 'PYTHON_EXECUTABLE'
or option_key == 'PYTHON_LIBRARY'
or option_key == 'PYTHON_LIBRARIES'
):
key = option_key + ":FILEPATH"
print(key)
elif option_key == 'PYTHON_INCLUDE_DIR':
key = key = option_key + ':PATH'
print(key)
else:
key = other_options[option_key]
if key not in paddle_build_options:
paddle_build_options[key] = option_value
options_process(args, paddle_build_options)
print("args:", args)
cmake_run(args, build_path)
# make
if only_cmake:
print(
"You have finished running cmake, the program exited,run 'ccmake build' to adjust build options and 'python setup.py install to build'"
)
sys.exit()
build_args = ["--build", ".", "--target", "install", "--config", 'Release']
max_jobs = os.getenv("MAX_JOBS")
if max_jobs is not None:
max_jobs = max_jobs or str(multiprocessing.cpu_count())

build_args += ["--"]
if IS_WINDOWS:
build_args += ["/p:CL_MPCount={}".format(max_jobs)]
else:
build_args += ["-j", max_jobs]
else:
build_args += ["-j", str(multiprocessing.cpu_count())]
environ_var = os.environ.copy()
build_run(build_args, build_path, environ_var)
run_cmake_build(build_path)


def get_setup_requires():
Expand Down