Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[icu] Enable parallel builds #6695

Merged
merged 30 commits into from
Jun 12, 2019
Merged

[icu] Enable parallel builds #6695

Show file tree
Hide file tree
Changes from 19 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
a18172a
Add VCPKG_NUM_LOGICAL_CORES and use it on a couple of ports
cbezault May 30, 2019
17e8bb9
Merge branch 'master' of https://github.com/Microsoft/vcpkg into numproc
cbezault May 30, 2019
387c1b9
Fix parallel build on windows
cbezault May 30, 2019
0b1f922
break out logic that retries running a command several times into its…
cbezault Jun 4, 2019
45bd845
Merge branch 'master' into numproc
cbezault Jun 4, 2019
7eb177f
add file I missed
cbezault Jun 4, 2019
1f8d18e
merge in master
cbezault Jun 4, 2019
81275fd
Merge branch 'numproc' of github.com:cbezault/vcpkg into numproc
cbezault Jun 4, 2019
2c24748
Update vcpkg_execute_build_process.cmake
cbezault Jun 4, 2019
c292c6f
trigger build
cbezault Jun 4, 2019
702bd1f
trigger rebuild
cbezault Jun 4, 2019
e0fd47c
rever fmilib changes
cbezault Jun 5, 2019
5a8f107
Merge branch 'numproc' of github.com:cbezault/vcpkg into numproc
cbezault Jun 5, 2019
e5337a4
rever fmilib changes
cbezault Jun 5, 2019
4fa926c
Merge branch 'master' into numproc
cbezault Jun 5, 2019
2bd1f22
Change name of VCPKG_NUM_LOGICAL_CORES to VCPKG_CONCURRENCY, add env …
cbezault Jun 6, 2019
acbe321
merge
cbezault Jun 6, 2019
37eb8bc
Merge branch 'numproc' of github.com:cbezault/vcpkg into numproc
cbezault Jun 6, 2019
174adac
Remove unecessary algorithm header
cbezault Jun 6, 2019
fb6e018
Merge branch 'master' of https://github.com/Microsoft/vcpkg into numproc
cbezault Jun 10, 2019
f1db83c
Some changes as requested by ras0219-msft
cbezault Jun 10, 2019
4e1a8b2
Merge branch 'numproc' of github.com:cbezault/vcpkg into numproc
cbezault Jun 10, 2019
79e7914
Fix whitespace change
cbezault Jun 10, 2019
00448ca
Add back the FS flag, accidentally deleted
cbezault Jun 10, 2019
2c13ca3
Fix curly brace typo
cbezault Jun 10, 2019
20b20f5
some fixes
cbezault Jun 11, 2019
4f315fd
Correctly apply patches
cbezault Jun 11, 2019
42ee7c4
Use vcpkg_extract_source_archive_ex
cbezault Jun 11, 2019
d2498a9
Use extract command correctly
cbezault Jun 12, 2019
05e73df
Add mt.exe message
cbezault Jun 12, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions ports/icu/fix_parallel_build_on_windows.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
diff --git a/source/data/Makefile.in b/source/data/Makefile.in
index 1140b69..936ef81 100644
--- a/source/data/Makefile.in
+++ b/source/data/Makefile.in
@@ -514,7 +514,7 @@ build-dir:
# The | is an order-only prerequisite. This helps when the -j option is used,
# and we don't want the files to be built before the directories are built.
ifneq ($(filter order-only,$(.FEATURES)),)
-$(ALL_FILES) $(ALL_INDEX_SRC_FILES): | build-dir
+$(ALL_FILES) $(ALL_INDEX_SRC_FILES) $(SO_VERSION_DATA): | build-dir
endif

# Now, sections for building each kind of data.
431 changes: 220 additions & 211 deletions ports/icu/portfile.cmake
View file
Open in desktop

Large diffs are not rendered by default.

120 changes: 4 additions & 116 deletions scripts/cmake/vcpkg_build_cmake.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,8 +75,6 @@ function(vcpkg_build_cmake)
endif()

message(STATUS "Building ${TARGET_TRIPLET}-${SHORT_BUILDTYPE}")
set(LOGPREFIX "${CURRENT_BUILDTREES_DIR}/${_bc_LOGFILE_ROOT}-${TARGET_TRIPLET}-${SHORT_BUILDTYPE}")
set(LOGS)

if(_bc_ADD_BIN_TO_PATH)
set(_BACKUP_ENV_PATH "$ENV{PATH}")
Expand All @@ -91,121 +89,11 @@ function(vcpkg_build_cmake)
set(ENV{PATH} "${CURRENT_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/bin${_PATHSEP}$ENV{PATH}")
endif()
endif()
execute_process(
vcpkg_execute_build_process(
COMMAND ${CMAKE_COMMAND} --build . --config ${CONFIG} ${TARGET_PARAM} -- ${BUILD_ARGS} ${PARALLEL_ARG}
cbezault marked this conversation as resolved.
Show resolved Hide resolved
OUTPUT_FILE "${LOGPREFIX}-out.log"
ERROR_FILE "${LOGPREFIX}-err.log"
RESULT_VARIABLE error_code
WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE})
if(error_code)
file(READ "${LOGPREFIX}-out.log" out_contents)
file(READ "${LOGPREFIX}-err.log" err_contents)

if(out_contents)
list(APPEND LOGS "${LOGPREFIX}-out.log")
endif()
if(err_contents)
list(APPEND LOGS "${LOGPREFIX}-err.log")
endif()

if(out_contents MATCHES "LINK : fatal error LNK1102:" OR out_contents MATCHES " fatal error C1060: ")
# The linker ran out of memory during execution. We will try continuing once more, with parallelism disabled.
message(STATUS "Restarting Build ${TARGET_TRIPLET}-${SHORT_BUILDTYPE} without parallelism because memory exceeded")
execute_process(
COMMAND ${CMAKE_COMMAND} --build . --config ${CONFIG} ${TARGET_PARAM} -- ${BUILD_ARGS} ${NO_PARALLEL_ARG}
OUTPUT_FILE "${LOGPREFIX}-out-1.log"
ERROR_FILE "${LOGPREFIX}-err-1.log"
RESULT_VARIABLE error_code
WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE})

if(error_code)
file(READ "${LOGPREFIX}-out-1.log" out_contents)
file(READ "${LOGPREFIX}-err-1.log" err_contents)

if(out_contents)
list(APPEND LOGS "${LOGPREFIX}-out-1.log")
endif()
if(err_contents)
list(APPEND LOGS "${LOGPREFIX}-err-1.log")
endif()
endif()
elseif(out_contents MATCHES ": No such file or directory")
# WSL workaround - WSL occassionally fails with no such file or directory. Detect if we are running in WSL and restart.
execute_process(COMMAND "uname" "-r"
OUTPUT_VARIABLE UNAME_R ERROR_VARIABLE UNAME_R
OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE)

if (UNAME_R MATCHES "Microsoft")
set(ITERATION 0)
while (ITERATION LESS 10 AND out_contents MATCHES ": No such file or directory")
MATH(EXPR ITERATION "${ITERATION}+1")
message(STATUS "Restarting Build ${TARGET_TRIPLET}-${SHORT_BUILDTYPE} because of wsl subsystem issue. Iteration: ${ITERATION}")
execute_process(
COMMAND ${CMAKE_COMMAND} --build . --config ${CONFIG} ${TARGET_PARAM} -- ${BUILD_ARGS}
OUTPUT_FILE "${LOGPREFIX}-out-${ITERATION}.log"
ERROR_FILE "${LOGPREFIX}-err-${ITERATION}.log"
RESULT_VARIABLE error_code
WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE})

if(error_code)
file(READ "${LOGPREFIX}-out-${ITERATION}.log" out_contents)
file(READ "${LOGPREFIX}-err-${ITERATION}.log" err_contents)

if(out_contents)
list(APPEND LOGS "${LOGPREFIX}-out-${ITERATION}.log")
endif()
if(err_contents)
list(APPEND LOGS "${LOGPREFIX}-err-${ITERATION}.log")
endif()
else()
break()
endif()
endwhile()
endif()
elseif(out_contents MATCHES "mt : general error c101008d: " OR out_contents MATCHES "mt.exe : general error c101008d: ")
# Antivirus workaround - occasionally files are locked and cause mt.exe to fail
set(ITERATION 0)
while (ITERATION LESS 3 AND (out_contents MATCHES "mt : general error c101008d: " OR out_contents MATCHES "mt.exe : general error c101008d: "))
MATH(EXPR ITERATION "${ITERATION}+1")
message(STATUS "Restarting Build ${TARGET_TRIPLET}-${SHORT_BUILDTYPE} because of mt.exe file locking issue. Iteration: ${ITERATION}")
execute_process(
COMMAND ${CMAKE_COMMAND} --build . --config ${CONFIG} ${TARGET_PARAM} -- ${BUILD_ARGS} ${PARALLEL_ARG}
OUTPUT_FILE "${LOGPREFIX}-out-${ITERATION}.log"
ERROR_FILE "${LOGPREFIX}-err-${ITERATION}.log"
RESULT_VARIABLE error_code
WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE})

if(error_code)
file(READ "${LOGPREFIX}-out-${ITERATION}.log" out_contents)
file(READ "${LOGPREFIX}-err-${ITERATION}.log" err_contents)

if(out_contents)
list(APPEND LOGS "${LOGPREFIX}-out-${ITERATION}.log")
endif()
if(err_contents)
list(APPEND LOGS "${LOGPREFIX}-err-${ITERATION}.log")
endif()
else()
break()
endif()
endwhile()
endif()

if(error_code)
set(STRINGIFIED_LOGS)
foreach(LOG ${LOGS})
file(TO_NATIVE_PATH "${LOG}" NATIVE_LOG)
list(APPEND STRINGIFIED_LOGS " ${NATIVE_LOG}\n")
endforeach()
set(_eb_COMMAND ${CMAKE_COMMAND} --build . --config ${CONFIG} ${TARGET_PARAM} -- ${BUILD_ARGS} ${PARALLEL_ARG})
set(_eb_WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE})
message(FATAL_ERROR
" Command failed: ${_eb_COMMAND}\n"
" Working Directory: ${_eb_WORKING_DIRECTORY}\n"
" See logs for more information:\n"
${STRINGIFIED_LOGS})
endif()
endif()
WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${SHORT_BUILDTYPE}
LOGNAME "${_bc_LOGFILE_ROOT}-${TARGET_TRIPLET}-${SHORT_BUILDTYPE}"
)
if(_bc_ADD_BIN_TO_PATH)
set(ENV{PATH} "${_BACKUP_ENV_PATH}")
endif()
Expand Down
1 change: 1 addition & 0 deletions scripts/cmake/vcpkg_common_functions.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ include(vcpkg_extract_source_archive)
include(vcpkg_extract_source_archive_ex)
include(vcpkg_execute_required_process)
include(vcpkg_execute_required_process_repeat)
include(vcpkg_execute_build_process)
include(vcpkg_find_acquire_program)
include(vcpkg_fixup_cmake_targets)
include(vcpkg_from_github)
Expand Down
101 changes: 101 additions & 0 deletions scripts/cmake/vcpkg_execute_build_process.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
# Usage: vcpkg_execute_build_process(COMMAND <cmd> [<args>...] WORKING_DIRECTORY </path/to/dir> LOGNAME <log_name>)
function(vcpkg_execute_build_process)
cmake_parse_arguments(_ebp "" "WORKING_DIRECTORY;LOGNAME" "COMMAND" ${ARGN})
set(LOG_OUT "${CURRENT_BUILDTREES_DIR}/${_ebp_LOGNAME}-out.log")
set(LOG_ERR "${CURRENT_BUILDTREES_DIR}/${_ebp_LOGNAME}-err.log")
execute_process(
COMMAND ${_ebp_COMMAND}
WORKING_DIRECTORY ${_ebp_WORKING_DIRECTORY}
OUTPUT_FILE ${LOG_OUT}
ERROR_FILE ${LOG_ERR}
RESULT_VARIABLE error_code
)

if(error_code)
file(READ ${LOG_OUT} out_contents)
file(READ ${LOG_ERR} err_contents)

if(out_contents)
list(APPEND LOGS ${LOG_OUT})
endif()
if(err_contents)
list(APPEND LOGS ${LOG_ERR})
endif()

if(out_contents MATCHES "LINK : fatal error LNK1102:" OR out_contents MATCHES " fatal error C1060: "
OR err_contents MATCHES "LINK : fatal error LNK1102:" OR err_contents MATCHES " fatal error C1060: ")
# The linker ran out of memory during execution. We will try continuing once more, with parallelism disabled.
message(STATUS "Restarting Build without parallelism because memory exceeded")
set(LOG_OUT "${CURRENT_BUILDTREES_DIR}/${_ebp_LOGNAME}-out-1.log")
set(LOG_ERR "${CURRENT_BUILDTREES_DIR}/${_ebp_LOGNAME}-err-1.log")
execute_process(
COMMAND ${_ebp_COMMAND}
WORKING_DIRECTORY ${_ebp_WORKING_DIRECTORY}
OUTPUT_FILE ${LOG_OUT}
ERROR_FILE ${LOG_ERR}
RESULT_VARIABLE error_code
)

cbezault marked this conversation as resolved.
Show resolved Hide resolved
if(error_code)
file(READ ${LOG_OUT} out_contents)
file(READ ${LOG_ERR} err_contents)

if(out_contents)
list(APPEND LOGS ${LOG_OUT)
endif()
if(err_contents)
list(APPEND LOGS ${LOG_ERR})
endif()
endif()
elseif(out_contents MATCHES ": No such file or directory" OR err_contents MATCHES ": No such file or directory")
# WSL workaround - WSL occassionally fails with no such file or directory. Detect if we are running in WSL and restart.
cbezault marked this conversation as resolved.
Show resolved Hide resolved
execute_process(COMMAND "uname" "-r"
OUTPUT_VARIABLE UNAME_R ERROR_VARIABLE UNAME_R
OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE)

if (UNAME_R MATCHES "Microsoft")
set(ITERATION 0)
while (ITERATION LESS 20 AND out_contents MATCHES ": No such file or directory" OR err_contents MATCHES ": No such file or directory")
MATH(EXPR ITERATION "${ITERATION}+1")
message(STATUS "Restarting Build because of wsl subsystem issue. Iteration: ${ITERATION}")
set(LOG_OUT "${CURRENT_BUILDTREES_DIR}/${_ebp_LOGNAME}-out-${ITERATION}.log")
set(LOG_ERR "${CURRENT_BUILDTREES_DIR}/${_ebp_LOGNAME}-err-${ITERATION}.log")
execute_process(
COMMAND ${_ebp_COMMAND}
WORKING_DIRECTORY ${_ebp_WORKING_DIRECTORY}
OUTPUT_FILE ${LOG_OUT}
ERROR_FILE ${LOG_ERR}
RESULT_VARIABLE error_code
)

if(error_code)
file(READ ${LOG_OUT} out_contents)
file(READ ${LOG_ERR} err_contents)

if(out_contents)
list(APPEND LOGS ${LOG_OUT})
endif()
if(err_contents)
list(APPEND LOGS ${LOG_ERR})
endif()
else()
break()
endif()
endwhile()
endif()
endif()

if(error_code)
set(STRINGIFIED_LOGS)
foreach(LOG ${LOGS})
file(TO_NATIVE_PATH "${LOG}" NATIVE_LOG)
list(APPEND STRINGIFIED_LOGS " ${NATIVE_LOG}\n")
endforeach()
message(FATAL_ERROR
" Command failed: ${_ebp_COMMAND}\n"
" Working Directory: ${_ebp_WORKING_DIRECTORY}\n"
" See logs for more information:\n"
${STRINGIFIED_LOGS})
endif(error_code)
endif(error_code)
endfunction(vcpkg_execute_build_process)
2 changes: 2 additions & 0 deletions toolsrc/include/vcpkg/base/system.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,6 @@ namespace vcpkg::System
const Optional<fs::path>& get_program_files_32_bit();

const Optional<fs::path>& get_program_files_platform_bitness();

int get_num_logical_cores();
}
5 changes: 5 additions & 0 deletions toolsrc/src/vcpkg/base/system.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -595,6 +595,11 @@ namespace vcpkg
#else
void System::register_console_ctrl_handler() {}
#endif

int System::get_num_logical_cores()
{
return std::thread::hardware_concurrency();
}
}

namespace vcpkg::Debug
Expand Down
18 changes: 18 additions & 0 deletions toolsrc/src/vcpkg/build.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -334,6 +334,23 @@ namespace vcpkg::Build
return ret;
}

static int get_concurrency()
cbezault marked this conversation as resolved.
Show resolved Hide resolved
{
int concurrency;

auto user_defined_concurrency = System::get_environment_variable("VCPKG_MAX_CONCURRENCY");
if (user_defined_concurrency)
{
concurrency = std::stoi(user_defined_concurrency.value_or_exit(VCPKG_LINE_INFO));
}
else
{
concurrency = System::get_num_logical_cores() + 1;
}

return concurrency;
}

static ExtendedBuildResult do_build_package(const VcpkgPaths& paths,
const PreBuildInfo& pre_build_info,
const PackageSpec& spec,
Expand Down Expand Up @@ -373,6 +390,7 @@ namespace vcpkg::Build
{"_VCPKG_DOWNLOAD_TOOL", to_string(config.build_package_options.download_tool)},
{"FEATURES", Strings::join(";", config.feature_list)},
{"ALL_FEATURES", all_features},
{"VCPKG_CONCURRENCY", std::to_string(get_concurrency())},
};

if (!System::get_environment_variable("VCPKG_FORCE_SYSTEM_BINARIES").has_value())
Expand Down