Make vendored libraries optional (#4544)

.github/workflows/win.yml

@@ -37,7 +37,7 @@ defaults:
 env:
   BUILD_TYPE: Release
   MSVC_VERSION: '2022'
-  VCPKG_VERSION: '2024.01.12'
+  VCPKG_VERSION: '8040303'
   VCPKG_INSTALL_OPTIONS: --x-abi-tools-use-exact-versions
   VCPKG_DISABLE_COMPILER_TRACKING: ON
 
@@ -61,33 +61,33 @@ jobs:
 
       - name: Cache vcpkg
         id: cache-vcpkg
-        uses: actions/cache@v4
+        uses: actions/cache@v3
         with:
           path: ${{ env.VCPKG_ROOT }}
-          key: vcpkg=${{ env.VCPKG_VERSION }}-msvc=${{ env.MSVC_VERSION }}-json=${{ hashFiles('vcpkg.json') }}-v2
-          save-always: 'true'
+          key: vcpkg=${{ env.VCPKG_VERSION }}-msvc=${{ env.MSVC_VERSION }}-json=${{ hashFiles('vcpkg.json') }}-v3
+          enableCrossOsArchive: true
 
       - name: Install GNU make & awk
         run: choco install gawk make
 
       - if: ${{ steps.cache-vcpkg.outputs.cache-hit != 'true' }}
         name: Bootstrap vcpkg and install packages (if cache miss)
         run: |
-          git clone --depth=1 --branch "$VCPKG_VERSION" https://github.com/microsoft/vcpkg.git
+          git clone https://github.com/microsoft/vcpkg.git
           cd vcpkg
+          git checkout $VCPKG_VERSION
           mkdir archives
           mkdir "$VCPKG_OVERLAY_TRIPLETS"
           TRIPLET_FILE="$VCPKG_OVERLAY_TRIPLETS/$VCPKG_DEFAULT_TRIPLET.cmake"
           cp triplets/x64-windows.cmake "$TRIPLET_FILE"
           echo "set(VCPKG_BUILD_TYPE release)" >> "$TRIPLET_FILE"
           echo "set(VCPKG_DISABLE_COMPILER_TRACKING $VCPKG_DISABLE_COMPILER_TRACKING)" >> "$TRIPLET_FILE"
           cmd.exe /c bootstrap-vcpkg.bat
-          vcpkg.exe \
+          vcpkg.exe install \
             --vcpkg-root="$VCPKG_ROOT" \
             --overlay-triplets="$VCPKG_OVERLAY_TRIPLETS" \
             --triplet="$VCPKG_DEFAULT_TRIPLET" \
-            $VCPKG_INSTALL_OPTIONS \
-            install
+            $VCPKG_INSTALL_OPTIONS
 
       - name: Setup Developer Command Prompt for VS
         uses: ilammy/msvc-dev-cmd@v1
@@ -110,7 +110,8 @@ jobs:
             -DENABLE_TESTS=OFF \
             -DENABLE_CCACHE=OFF \
             -DENABLE_SERVICES=OFF \
-            -DENABLE_BENCHMARKS=OFF
+            -DENABLE_BENCHMARKS=OFF \
+            -DPREFER_EXTERNAL=ON \
 
       - name: Build Valhalla
         shell: powershell

.gitmodules

@@ -37,9 +37,6 @@
 [submodule "third_party/cpp-statsd-client"]
 	path = third_party/cpp-statsd-client
 	url = https://github.com/vthiery/cpp-statsd-client
-[submodule "third_party/lz4"]
-	path = third_party/lz4
-	url = https://github.com/lz4/lz4
 [submodule "third_party/cxxopts"]
 	path = third_party/cxxopts
 	url = https://github.com/jarro2783/cxxopts.git

CHANGELOG.md

@@ -84,6 +84,7 @@
    * CHANGED: refactored matrix code to include a base class for all matrix algorithms to prepare for second passes on matrix [#4535](https://github.com/valhalla/valhalla/pull/4535)
    * ADDED: matrix second pass for connections not found in the first pass, analogous to /route [#4536](https://github.com/valhalla/valhalla/pull/4536)
    * UPDATED: cxxopts to 3.1.1 [#4541](https://github.com/valhalla/valhalla/pull/4541)
+   * CHANGED: make use of vendored libraries optional (other than libraries which are not commonly in package managers or only used for testing) [#4544](https://github.com/valhalla/valhalla/pull/4544)
 
 ## Release Date: 2023-05-11 Valhalla 3.4.0
 * **Removed**

CMakeLists.txt

@@ -34,6 +34,7 @@ option(ENABLE_WERROR "Convert compiler warnings to errors. Requires ENABLE_COMPI
 option(ENABLE_BENCHMARKS "Enable microbenchmarking" ON)
 option(ENABLE_THREAD_SAFE_TILE_REF_COUNT "If ON uses shared_ptr as tile reference(i.e. it is thread safe)" OFF)
 option(ENABLE_SINGLE_FILES_WERROR "Convert compiler warnings to errors for single files" ON)
+option(PREFER_EXTERNAL_DEPS "Whether to use internally vendored headers or find the equivalent external package" OFF)
 # useful to workaround issues likes this https://stackoverflow.com/questions/24078873/cmake-generated-xcode-project-wont-compile
 option(ENABLE_STATIC_LIBRARY_MODULES "If ON builds Valhalla modules as STATIC library targets" OFF)
 
@@ -99,6 +100,7 @@ include(ValhallaSourceGroups)
 find_package(PkgConfig REQUIRED)
 find_package(Threads REQUIRED)
 pkg_check_modules(ZLIB REQUIRED IMPORTED_TARGET zlib)
+pkg_check_modules(LZ4 REQUIRED IMPORTED_TARGET liblz4)
 
 # cURL
 set(curl_targets "")
@@ -118,9 +120,55 @@ add_definitions(-DBOOST_NO_CXX11_SCOPED_ENUMS)
 add_definitions(-DBOOST_ALLOW_DEPRECATED_HEADERS)
 add_definitions(-DBOOST_BIND_GLOBAL_PLACEHOLDERS)
 
+# resolve vendored libraries
+set(date_include_dir ${VALHALLA_SOURCE_DIR}/third_party/date/include)
+set(rapidjson_include_dir ${CMAKE_SOURCE_DIR}/third_party/rapidjson/include)
+set(robinhoodhashing_include_dir ${CMAKE_SOURCE_DIR}/third_party/robin-hood-hashing/src/include)
+set(cxxopts_include_dir ${VALHALLA_SOURCE_DIR}/third_party/cxxopts/include)
+set(dirent_include_dir ${CMAKE_SOURCE_DIR}/third_party/dirent/include)
+if (PREFER_EXTERNAL_DEPS)
+  # date
+  find_package(date QUIET)
+  if (date_FOUND)
+    get_target_property(date_include_dir date::date INTERFACE_INCLUDE_DIRECTORIES)
+  else()
+    message(WARNING "No date found in system libraries, using vendored date...")
+  endif()
+  # rapidjson
+  find_package(RapidJSON QUIET)
+  if (RapidJSON_FOUND)
+    get_target_property(rapidjson_include_dir rapidjson INTERFACE_INCLUDE_DIRECTORIES)
+  else()
+    message(WARNING "No RapidJSON found in system libraries, using vendored RapidJSON...")
+  endif()
+  # robin-hood-hashing
+  find_package(robin_hood QUIET)
+  if (robin_hood_FOUND)
+    get_target_property(robinhoodhashing_include_dir robin_hood::robin_hood INTERFACE_INCLUDE_DIRECTORIES)
+  else()
+    message(WARNING "No robin_hood found in system libraries, using vendored robin_hood...")
+  endif()
+  # cxxopts
+  if (ENABLE_DATA_TOOLS OR ENABLE_TOOLS)
+    find_package(cxxopts QUIET)
+    if (cxxopts_FOUND)
+      get_target_property(cxxopts_include_dir cxxopts::cxxopts INTERFACE_INCLUDE_DIRECTORIES)
+    else()
+      message(WARNING "No cxxopts found in system libraries, using vendored cxxopts...")
+    endif()
+  endif()
+  # dirent
+  if (WIN32)
+    find_path(dirent_include_dir dirent.h REQUIRED)
+    if (dirent_include_dir-NOTFOUND)
+      message(WARNING "No dirent.h found in system headers, using vendored dirent.h...")
+    endif()
+  endif()
+endif()
+
 # Protobuf is non-trivial to include via pkg-config, pkg_check_modules has no way to check
 # for protoc location in a platform agnostic manner
-# prefer CONFIG mode over MODULE mode, there were a few CMake/protobuf messups
+# prefer CONFIG mode over MODULE mode, which versions configuration on the package, not CMake
 # NOTE: this is only supported for cmake >= 3.15, but shouldn't be a problem in real life
 set(CMAKE_FIND_PACKAGE_PREFER_CONFIG ON)
 # newer protobuf versions require a compat bool
@@ -211,7 +259,7 @@ if(ENABLE_TOOLS)
     set_target_properties(${program} PROPERTIES FOLDER "Tools")
     create_source_groups("Source Files" ${path})
     target_link_libraries(${program} valhalla)
-    target_include_directories(${program} PRIVATE ${VALHALLA_SOURCE_DIR}/third_party/cxxopts/include)
+    target_include_directories(${program} PRIVATE ${cxxopts_include_dir})
     install(TARGETS ${program} DESTINATION "${CMAKE_INSTALL_BINDIR}" COMPONENT runtime)
   endforeach()
 endif()
@@ -222,7 +270,7 @@ if(ENABLE_DATA_TOOLS)
     add_executable(${program} ${path})
     create_source_groups("Source Files" ${path})
     set_target_properties(${program} PROPERTIES FOLDER "Data Tools")
-    target_include_directories(${program} PRIVATE ${VALHALLA_SOURCE_DIR}/third_party/cxxopts/include)
+    target_include_directories(${program} PRIVATE ${cxxopts_include_dir})
     target_link_libraries(${program} valhalla)
     if (LuaJIT_FOUND AND APPLE AND CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "x86_64")
       # Using LuaJIT on macOS on Intel processors requires a couple of extra linker flags
@@ -272,12 +320,12 @@ install(FILES COPYING CHANGELOG.md
   COMPONENT runtime)
 
 # install third_party
-install(DIRECTORY ${CMAKE_SOURCE_DIR}/third_party/rapidjson/include/
+install(DIRECTORY ${rapidjson_include_dir}/
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/valhalla/third_party")
-install(DIRECTORY ${CMAKE_SOURCE_DIR}/third_party/date/include/
+install(DIRECTORY ${date_include_dir}/
   DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/valhalla/third_party")
 if (WIN32)
-  install(DIRECTORY ${CMAKE_SOURCE_DIR}/third_party/dirent/include/
+  install(DIRECTORY ${dirent_include_dir}/
     DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/valhalla/third_party")
 endif()
 

README.md

@@ -20,7 +20,7 @@ Valhalla is an open source routing engine and accompanying libraries for use wit
 
 | Linux/MacOs | Windows | Code Coverage |
 | ----------- | ------- | ------------- |
-| [![Circle CI](https://circleci.com/gh/valhalla/valhalla/tree/master.svg?style=svg)](https://circleci.com/gh/valhalla/valhalla/tree/master) | [![Build Status](https://dev.azure.com/valhalla1/valhalla/_apis/build/status/valhalla.valhalla?branchName=master)](https://dev.azure.com/valhalla1/valhalla/_build/latest?definitionId=1&branchName=master) | [![codecov](https://codecov.io/gh/valhalla/valhalla/branch/master/graph/badge.svg)](https://codecov.io/gh/valhalla/valhalla) |
+| [![Circle CI](https://circleci.com/gh/valhalla/valhalla/tree/master.svg?style=svg)](https://circleci.com/gh/valhalla/valhalla/tree/master) | [![Windows CI](https://github.com/valhalla/valhalla/actions/workflows/win.yml/badge.svg)](https://github.com/valhalla/valhalla/actions/workflows/win.yml) | [![codecov](https://codecov.io/gh/valhalla/valhalla/branch/master/graph/badge.svg)](https://codecov.io/gh/valhalla/valhalla) |
 
 
 ## License

docs/docs/building.md

@@ -26,11 +26,31 @@ Important build options include:
 | `-DENABLE_SANITIZERS` (`ON` / `OFF`) | Build with all the integrated sanitizers (defaults to off).|
 | `-DENABLE_ADDRESS_SANITIZER` (`ON` / `OFF`) | Build with address sanitizer (defaults to off).|
 | `-DENABLE_UNDEFINED_SANITIZER` (`ON` / `OFF`) | Build with undefined behavior sanitizer (defaults to off).|
+| `-DPREFER_SYSTEM_DEPS` (`ON` / `OFF`) | Whether to use internally vendored headers or find the equivalent external package (defaults to off).|
 
-If you're building on Apple Silicon and use the Rosetta terminal (see below), you might need to additionally specify the appropriate options:
+### Building with `vcpkg` - any platform
 
-```
-cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_OSX_ARCHITECTURES="x86_64"
+Instead of installing the dependencies system-wide, you can also opt to use [`vcpkg`](https://github.com/microsoft/vcpkg).
+
+The following commands should work on all platforms:
+
+```bash
+git clone --recurse-submodules https://github.com/valhalla/valhalla
+cd valhalla
+git clone https://github.com/microsoft/vcpkg && git -C vcpkg checkout <some-tag>
+./vcpkg/bootstrap-vcpkg.sh
+# windows: cmd.exe /c bootstrap-vcpkg.bat
+# only build Release versions of dependencies, not Debug
+echo "VCPKG_BUILD_TYPE release" >> vcpkg/triplets/x64-linux.cmake
+# windows: echo.set(VCPKG_BUILD_TYPE release)>> .\vcpkg\triplets\x64-windows.cmake
+# osx: echo "VCPKG_BUILD_TYPE release" >> vcpkg/triplets/arm64-osx.cmake
+./vcpkg/vcpkg install
+
+# if you want to ENABLE_SERVICES=ON, install https://github.com/kevinkreiser/prime_server#build-and-install (no Windows)
+cmake -B build -DCMAKE_TOOLCHAIN_FILE=$PWD/vcpkg/scripts/buildsystems/vcpkg.cmake -DENABLE_SERVICE=OFF
+cmake --build build -- -j$(nproc)
+# windows: cmake --build build --config Release -- /clp:ErrorsOnly /p:BuildInParallel=true /m:4
+# osx: cmake --build build -- -j$(sysctl -n hw.physicalcpu)
 ```
 
 ### Building from Source - Linux
@@ -52,34 +72,9 @@ sudo make -C build install
 
 ### Building from Source - macOS
 
-#### Configuring Rosetta for ARM64 MacBook
-
-Check your architecture typing `arch` in the terminal. In case the result is `arm64` set up Rosetta terminal to emulate x86_64 behavior. Otherwise, skip this step.
-
-1. Go to `Finder > Application > Utilities`.
-2. Select `Terminal` and right-click on it, then choose `Duplicate`.
-3. Rename the duplicated app `Rosetta Terminal`.
-4. Now select `Rosetta Terminal` application, right-click and choose `Get Info` .
-5. Check the box for `Open using Rosetta`, then close the `Get Info` window.
-6. Make sure you get `i386` after typing `arch` command in  `Rosetta Terminal`.
-7. Now it fully supports Homebrew and other x86_64 command line applications.
-
-Install [Homebrew](http://brew.sh) in the `Rosetta Terminal` app and update the aliases.
-
-```
-echo "alias ibrew='arch -x86_64 /usr/local/bin/brew'" >> ~/.zshrc
-echo "alias mbrew='arch -arm64e /opt/homebrew/bin/brew'" >> ~/.zshrc
-```
-
-You will use them to specify the platform when installing a library. Note: use `ibrew` in `Rosetta Terminal` to install all dependencies for `valhalla` and `prime_server` projects.
-
-**_NOTE:_** If when installing packages below you get message `attempting to link with file built for macOS-arm64`, you can remove already installed packages for arm64 i.e. `mbrew uninstall ...`. Also, if there are problems with individual packages, you can install them from sources e.g. [geos](https://github.com/libgeos/geos) or [sqlite](https://www.sqlite.org/download.html).
-
-**_NOTE:_** It is possible to build Valhalla natively for Apple Silicon, but some dependencies(e.g. LuaJIT) don't have stable versions supporting Apple Silicon and have to be built and installed manually from source.
-
-#### Installing dependencies
+Both arm64 and x64 should build cleanly with the below commands.
 
-To install valhalla on macOS, you need to install its dependencies with [Homebrew](http://brew.sh):
+To install Valhalla on macOS, you need to install its dependencies with [Homebrew](http://brew.sh):
 
 ```bash
 # install dependencies (automake & czmq are required by prime_server)

src/CMakeLists.txt

@@ -186,10 +186,10 @@ target_include_directories(valhalla
     ${VALHALLA_SOURCE_DIR}/valhalla # TODO: this path must be removed and changed to #include <valhalla/...> in headers
     ${VALHALLA_BINARY_DIR}
     ${VALHALLA_BINARY_DIR}/valhalla  # TODO: this path must be removed and changed to #include <valhalla/...> in headers
-    ${VALHALLA_SOURCE_DIR}/third_party/rapidjson/include
-    ${VALHALLA_SOURCE_DIR}/third_party/date/include
+    ${rapidjson_include_dir}
+    ${date_include_dir}
     ${VALHALLA_SOURCE_DIR}/third_party/cpp-statsd-client/include
-    $<$<BOOL:${WIN32}>:${VALHALLA_SOURCE_DIR}/third_party/dirent/include>
+    $<$<BOOL:${WIN32}>:${dirent_include_dir}>
   PRIVATE
     ${libvalhalla_include_directories})
 

src/baldr/CMakeLists.txt

@@ -38,14 +38,10 @@ set(includes
      ${CMAKE_CURRENT_BINARY_DIR}/src/baldr
      )
 
-# treat date library as system
 set(system_includes 
-  ${VALHALLA_SOURCE_DIR}/third_party/date/include
-  $<$<BOOL:${WIN32}>:${VALHALLA_SOURCE_DIR}/third_party/dirent/include>
-  ${VALHALLA_SOURCE_DIR}/third_party/rapidjson/include)
-if(APPLE)
-  list(APPEND system_includes ${VALHALLA_SOURCE_DIR}/third_party/date/include/date)
-endif()
+  ${date_include_dir}
+  $<$<BOOL:${WIN32}>:${dirent_include_dir}>
+  ${rapidjson_include_dir})
 
 set(sources
     accessrestriction.cc

src/bindings/python/CMakeLists.txt

@@ -1,4 +1,13 @@
-add_subdirectory(${VALHALLA_SOURCE_DIR}/third_party/pybind11 ${CMAKE_BINARY_DIR}/third_party/pybind11)
+if (PREFER_EXTERNAL_DEPS)
+  find_package(pybind11 QUIET)
+  if (NOT pybind11_FOUND)
+    message(WARNING "No pybind11 found in system libraries, using vendored pybind11...")
+  endif()
+endif()
+
+if (NOT TARGET pybind11::pybind11_headers)
+  add_subdirectory(${VALHALLA_SOURCE_DIR}/third_party/pybind11 ${CMAKE_BINARY_DIR}/third_party/pybind11)
+endif()
 
 pybind11_add_module(python_valhalla python.cc)
 target_link_libraries(python_valhalla PUBLIC valhalla)

src/loki/CMakeLists.txt

@@ -21,13 +21,9 @@ set(sources_with_warnings
   trace_route_action.cc
   worker.cc)
 
-# treat date library as system
 set(system_includes 
-  ${VALHALLA_SOURCE_DIR}/third_party/date/include
-  $<$<BOOL:${WIN32}>:${VALHALLA_SOURCE_DIR}/third_party/dirent/include>)
-if(APPLE)
-  list(APPEND system_includes ${VALHALLA_SOURCE_DIR}/third_party/date/include/date)
-endif()
+  ${date_include_dir}
+  $<$<BOOL:${WIN32}>:${dirent_include_dir}>)
 
 valhalla_module(NAME loki
   SOURCES ${sources}
@@ -38,7 +34,7 @@ valhalla_module(NAME loki
       ${VALHALLA_SOURCE_DIR}
       ${VALHALLA_SOURCE_DIR}/valhalla
     PRIVATE
-      ${VALHALLA_SOURCE_DIR}/third_party/rapidjson/include
+      ${rapidjson_include_dir}
       ${CMAKE_BINARY_DIR}
   SYSTEM_INCLUDE_DIRECTORIES
     PUBLIC

src/meili/CMakeLists.txt

@@ -14,13 +14,9 @@ set(sources_with_warnings
   transition_cost_model.cc
   viterbi_search.cc)
 
-# treat date library as system
 set(system_includes 
-  ${VALHALLA_SOURCE_DIR}/third_party/date/include
-  $<$<BOOL:${WIN32}>:${VALHALLA_SOURCE_DIR}/third_party/dirent/include>)
-if(APPLE)
-  list(APPEND system_includes ${VALHALLA_SOURCE_DIR}/third_party/date/include/date)
-endif()
+  ${date_include_dir}
+  $<$<BOOL:${WIN32}>:${dirent_include_dir}>)
 
 valhalla_module(NAME meili
   SOURCES ${sources}
@@ -36,7 +32,7 @@ valhalla_module(NAME meili
     PUBLIC
       ${system_includes}
     PRIVATE
-      ${VALHALLA_SOURCE_DIR}/third_party/rapidjson/include
+      ${rapidjson_include_dir}
   DEPENDS
     valhalla::sif
     ${valhalla_protobuf_targets}

src/midgard/CMakeLists.txt

@@ -22,8 +22,8 @@ valhalla_module(NAME midgard
       ${VALHALLA_SOURCE_DIR}/valhalla
   SYSTEM_INCLUDE_DIRECTORIES
     PUBLIC
-      $<$<BOOL:${WIN32}>:${VALHALLA_SOURCE_DIR}/third_party/dirent/include>
+      $<$<BOOL:${WIN32}>:${dirent_include_dir}>
     PRIVATE
-      ${VALHALLA_SOURCE_DIR}/third_party/robin-hood-hashing/src/include
+      ${robinhoodhashing_include_dir}
   DEPENDS
     Boost::boost)

src/mjolnir/CMakeLists.txt

@@ -62,11 +62,10 @@ set(sources
   util.cc
   validatetransit.cc)
 
-# treat date library as system
 set(system_includes
-  ${VALHALLA_SOURCE_DIR}/third_party/date/include
-  ${VALHALLA_SOURCE_DIR}/third_party/rapidjson/include
-  $<$<BOOL:${WIN32}>:${VALHALLA_SOURCE_DIR}/third_party/dirent/include>
+  ${date_include_dir}
+  ${rapidjson_include_dir}
+  $<$<BOOL:${WIN32}>:${dirent_include_dir}>
   ${PROTOBUF_INCLUDE_DIR})
 if(APPLE)
   list(APPEND system_includes ${VALHALLA_SOURCE_DIR}/third_party/date/include/date)
@@ -87,7 +86,7 @@ valhalla_module(NAME mjolnir
       ${system_includes}
     PRIVATE
       ${VALHALLA_SOURCE_DIR}/third_party/just_gtfs/include
-      ${VALHALLA_SOURCE_DIR}/third_party/robin-hood-hashing/src/include
+      ${robinhoodhashing_include_dir}
   DEPENDS
     valhalla::proto
     valhalla::baldr