From 36bfa325bfa84bbcce917bb66fa1de9fd7bd5cce Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Sun, 10 Mar 2024 14:11:13 +0000 Subject: [PATCH 1/4] Pin Conan revisions correctly Conan dependencies are not pinned correctly. This means we're pulling in a newer onetbb recipe that no longer has a shared library option. Following other examples of how to pin revisions with cmake for conan v1, we correctly pin the expected revisions. Longer term we should look into - upgrading to conan v2 - defining the conan config separately from cmakelists.txt - understanding the need for disabling onetbb shared library support but for the purposes of reviving CI, this will be sufficient. --- .github/workflows/osrm-backend.yml | 4 +- CMakeLists.txt | 69 +++++++++++++++++------------- 2 files changed, 41 insertions(+), 32 deletions(-) diff --git a/.github/workflows/osrm-backend.yml b/.github/workflows/osrm-backend.yml index fb29e5cb87b..385f9d18ee7 100644 --- a/.github/workflows/osrm-backend.yml +++ b/.github/workflows/osrm-backend.yml @@ -369,9 +369,9 @@ jobs: uses: actions/cache@v3 with: path: ~/.conan - key: v6-conan-${{ matrix.name }}-${{ github.sha }} + key: v7-conan-${{ matrix.name }}-${{ github.sha }} restore-keys: | - v6-conan-${{ matrix.name }}- + v7-conan-${{ matrix.name }}- - name: Enable test cache uses: actions/cache@v3 with: diff --git a/CMakeLists.txt b/CMakeLists.txt index b97ca3e6803..cf18d6ac43a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -151,7 +151,7 @@ add_library(UPDATER OBJECT ${UpdaterGlob}) add_library(STORAGE OBJECT ${StorageGlob}) add_library(ENGINE OBJECT ${EngineGlob}) -if (BUILD_ROUTED) +if (BUILD_ROUTED) add_library(SERVER OBJECT ${ServerGlob}) add_executable(osrm-routed src/tools/routed.cpp $ $) endif() @@ -312,7 +312,7 @@ add_subdirectory(${FLATBUFFERS_SRC_DIR} set(FMT_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/fmt-9.1.0/include") add_compile_definitions(FMT_HEADER_ONLY) include_directories(SYSTEM ${FMT_INCLUDE_DIR}) - + # see https://stackoverflow.com/questions/70898030/boost-link-error-using-conan-find-package if (MSVC) @@ -322,22 +322,24 @@ endif() if(ENABLE_CONAN) message(STATUS "Installing dependencies via Conan") - # Conan will generate Find*.cmake files to build directory, so we use them with the highest priority + # Conan will generate Find*.cmake files to build directory, so we use them with the highest priority list(INSERT CMAKE_MODULE_PATH 0 ${CMAKE_BINARY_DIR}) list(INSERT CMAKE_PREFIX_PATH 0 ${CMAKE_BINARY_DIR}) include(${CMAKE_CURRENT_LIST_DIR}/cmake/conan.cmake) - set(CONAN_BOOST_VERSION "1.79.0#96e4902111a2e343a8ba0aa95391bb58") - set(CONAN_BZIP2_VERSION "1.0.8#d1b2d5816f25865acf978501dff1f897") - set(CONAN_EXPAT_VERSION "2.2.10#916908d4a570ad839edd25322c3268cd") - set(CONAN_LUA_VERSION "5.4.4#3ec62efc37cd0a5d80b9e5cb35277360") - set(CONAN_TBB_VERSION "2021.3.0#507ec17cbd51a84167e143b20d170eea") - + conan_check(REQUIRED) + + set(CONAN_BOOST_VERSION "1.79.0@#96e4902111a2e343a8ba0aa95391bb58") + set(CONAN_BZIP2_VERSION "1.0.8@#d1b2d5816f25865acf978501dff1f897") + set(CONAN_EXPAT_VERSION "2.2.10@#916908d4a570ad839edd25322c3268cd") + set(CONAN_LUA_VERSION "5.4.4@#3ec62efc37cd0a5d80b9e5cb35277360") + set(CONAN_TBB_VERSION "2021.3.0@#507ec17cbd51a84167e143b20d170eea") + set(CONAN_SYSTEM_INCLUDES ON) - # TODO: - # if we link TBB dynamically osrm-extract.exe finishes on the first access to any TBB symbol + # TODO: + # if we link TBB dynamically osrm-extract.exe finishes on the first access to any TBB symbol # with exit code = -1073741515, which means that program cannot load required DLL. if (MSVC) set(TBB_SHARED False) @@ -345,14 +347,14 @@ if(ENABLE_CONAN) set(TBB_SHARED True) endif() - set(CONAN_ARGS - REQUIRES + set(CONAN_ARGS + REQUIRES "boost/${CONAN_BOOST_VERSION}" "bzip2/${CONAN_BZIP2_VERSION}" "expat/${CONAN_EXPAT_VERSION}" "lua/${CONAN_LUA_VERSION}" "onetbb/${CONAN_TBB_VERSION}" - BASIC_SETUP + BASIC_SETUP GENERATORS cmake_find_package json # json generator generates a conanbuildinfo.json in the build folder so (non-CMake) projects can easily parse OSRM's dependencies KEEP_RPATHS NO_OUTPUT_DIRS @@ -361,6 +363,13 @@ if(ENABLE_CONAN) boost:without_stacktrace=True # Apple Silicon cross-compilation fails without it BUILD missing ) + + # Enable revisions in the conan config + execute_process(COMMAND ${CONAN_CMD} config set general.revisions_enabled=1 RESULT_VARIABLE RET_CODE) + if(NOT "${RET_CODE}" STREQUAL "0") + message(FATAL_ERROR "Error setting revisions for Conan: '${RET_CODE}'") + endif() + # explicitly say Conan to use x86 dependencies if build for x86 platforms (https://github.com/conan-io/cmake-conan/issues/141) if(NOT CMAKE_SIZEOF_VOID_P EQUAL 8) conan_cmake_run("${CONAN_ARGS};ARCH;x86") @@ -389,7 +398,7 @@ if(ENABLE_CONAN) set(Boost_ZLIB_LIBRARY "${Boost_zlib_LIB_TARGETS}") set(Boost_REGEX_LIBRARY "${Boost_regex_LIB_TARGETS}") set(Boost_UNIT_TEST_FRAMEWORK_LIBRARY "${Boost_unit_test_framework_LIB_TARGETS}") - + find_package(BZip2 REQUIRED) find_package(EXPAT REQUIRED) @@ -740,23 +749,23 @@ if (ENABLE_FUZZING) add_subdirectory(fuzz) endif () -# add headers sanity check target that includes all headers independently -set(check_headers_dir "${PROJECT_BINARY_DIR}/check-headers") -file(GLOB_RECURSE headers_to_check - ${PROJECT_BINARY_DIR}/*.hpp - ${PROJECT_SOURCE_DIR}/include/*.hpp) -foreach(header ${headers_to_check}) +# add headers sanity check target that includes all headers independently +set(check_headers_dir "${PROJECT_BINARY_DIR}/check-headers") +file(GLOB_RECURSE headers_to_check + ${PROJECT_BINARY_DIR}/*.hpp + ${PROJECT_SOURCE_DIR}/include/*.hpp) +foreach(header ${headers_to_check}) if ("${header}" MATCHES ".*/include/nodejs/.*") # we do not check NodeJS bindings headers continue() endif() - get_filename_component(filename ${header} NAME_WE) - set(filename "${check_headers_dir}/${filename}.cpp") - if (NOT EXISTS ${filename}) - file(WRITE ${filename} "#include \"${header}\"\n") - endif() - list(APPEND sources ${filename}) -endforeach() -add_library(check-headers STATIC EXCLUDE_FROM_ALL ${sources}) -set_target_properties(check-headers PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${check_headers_dir}) + get_filename_component(filename ${header} NAME_WE) + set(filename "${check_headers_dir}/${filename}.cpp") + if (NOT EXISTS ${filename}) + file(WRITE ${filename} "#include \"${header}\"\n") + endif() + list(APPEND sources ${filename}) +endforeach() +add_library(check-headers STATIC EXCLUDE_FROM_ALL ${sources}) +set_target_properties(check-headers PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${check_headers_dir}) From 76c28a93014c4b1f35dd84323b94802efe44588b Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Thu, 14 Mar 2024 21:59:12 +0000 Subject: [PATCH 2/4] Fix macos CI builds --- .github/workflows/osrm-backend.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/osrm-backend.yml b/.github/workflows/osrm-backend.yml index 385f9d18ee7..fdebe1da47c 100644 --- a/.github/workflows/osrm-backend.yml +++ b/.github/workflows/osrm-backend.yml @@ -462,6 +462,11 @@ jobs: fi echo "CC=${CCOMPILER}" >> $GITHUB_ENV echo "CXX=${CXXCOMPILER}" >> $GITHUB_ENV + if [[ "${RUNNER_OS}" == "macOS" ]]; then + # missing from GCC path, needed for conan builds of libiconv for example. + sudo xcode-select --switch /Library/Developer/CommandLineTools + echo "LIBRARY_PATH=${LIBRARY_PATH}:/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib" >> $GITHUB_ENV + fi - name: Build and install OSRM run: | From bc6cf14e79e7e740458d804660f30a915cd80054 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Thu, 14 Mar 2024 23:30:29 +0000 Subject: [PATCH 3/4] Update osrm-backend.yml --- .github/workflows/osrm-backend.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/osrm-backend.yml b/.github/workflows/osrm-backend.yml index fdebe1da47c..bfad0846572 100644 --- a/.github/workflows/osrm-backend.yml +++ b/.github/workflows/osrm-backend.yml @@ -464,7 +464,8 @@ jobs: echo "CXX=${CXXCOMPILER}" >> $GITHUB_ENV if [[ "${RUNNER_OS}" == "macOS" ]]; then # missing from GCC path, needed for conan builds of libiconv for example. - sudo xcode-select --switch /Library/Developer/CommandLineTools + sudo xcode-select --switch /Library/Developer/CommandLineTools + ls -al /Library/Developer/CommandLineTools/SDKs echo "LIBRARY_PATH=${LIBRARY_PATH}:/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib" >> $GITHUB_ENV fi From 38ff187095eb5217a0b5bff9746c41b5da49b235 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Fri, 15 Mar 2024 07:35:58 +0000 Subject: [PATCH 4/4] Update osrm-backend.yml --- .github/workflows/osrm-backend.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/osrm-backend.yml b/.github/workflows/osrm-backend.yml index bfad0846572..389cb48586b 100644 --- a/.github/workflows/osrm-backend.yml +++ b/.github/workflows/osrm-backend.yml @@ -463,10 +463,10 @@ jobs: echo "CC=${CCOMPILER}" >> $GITHUB_ENV echo "CXX=${CXXCOMPILER}" >> $GITHUB_ENV if [[ "${RUNNER_OS}" == "macOS" ]]; then - # missing from GCC path, needed for conan builds of libiconv for example. + # missing from GCC path, needed for conan builds of libiconv, for example. sudo xcode-select --switch /Library/Developer/CommandLineTools - ls -al /Library/Developer/CommandLineTools/SDKs echo "LIBRARY_PATH=${LIBRARY_PATH}:/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib" >> $GITHUB_ENV + echo "CPATH=${CPATH}:/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/include" >> $GITHUB_ENV fi - name: Build and install OSRM