From c562d9fb5055d311bd700aeaeddab24f20e9c8d8 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Wed, 8 Mar 2023 17:24:14 -0600 Subject: [PATCH] Optionally build all dependencies with vcpkg This adds a `--enable-vcpkg` configuration flag to CMake that allows developers to build all dependencies with vcpkg. --- .github/workflows/ci-linux_mac.yml | 2 +- bootstrap | 5 + cmake/Modules/FindAWSSDK_EP.cmake | 37 +++++ cmake/Modules/FindMagic_EP.cmake | 34 +++++ cmake/Modules/FindOpenSSL_EP.cmake | 4 +- cmake/Modules/FindWebp_EP.cmake | 10 +- cmake/Options/BuildOptions.cmake | 1 + cmake/Options/TileDBToolchain.cmake | 30 +++- cmake/TileDB-Superbuild.cmake | 1 + test/external/src/absl_library_targets.cmake | 140 ++++++++++++++++++- test/external/src/absl_link_test.cc | 32 ++--- test/external/src/crc32_link_test.cc | 2 +- vcpkg-configuration.json | 3 + vcpkg.json | 77 +++++++++- 14 files changed, 348 insertions(+), 30 deletions(-) create mode 100644 vcpkg-configuration.json diff --git a/.github/workflows/ci-linux_mac.yml b/.github/workflows/ci-linux_mac.yml index f682ab50255..bc1d5d7f281 100644 --- a/.github/workflows/ci-linux_mac.yml +++ b/.github/workflows/ci-linux_mac.yml @@ -96,7 +96,7 @@ jobs: if: ${{ startsWith(matrix.os, 'macos-') == true }} run: | set -e pipefail - brew install pkg-config + brew install automake pkg-config - name: 'Configure libtiledb build' id: configure diff --git a/bootstrap b/bootstrap index 63ee63388a7..3c7f8317a27 100755 --- a/bootstrap +++ b/bootstrap @@ -34,6 +34,7 @@ Configuration: --help print this message --prefix=PREFIX install files in tree rooted at PREFIX ['"${default_prefix}"'] + --enable-vcpkg use vcpkg for downloading and building dependnecies --dependency=DIRs specify the dependencies at DIRs, separated by colon ['"${default_dependency}"'] --force-build-all-deps force building of all dependencies, even those @@ -84,6 +85,7 @@ Example: # Parse arguments prefix_dirs="${default_prefix}" +tiledb_vcpkg="OFF" dependency_dir="${default_dependency}" sanitizer="" build_type="Release" @@ -114,6 +116,7 @@ while test $# != 0; do case "$1" in --prefix=*) dir=`arg "$1"` prefix_dirs="$dir";; + --enable-vcpkg) tiledb_vcpkg="ON";; --dependency=*) dir=`arg "$1"` dependency_dir="$dir";; --force-build-all-deps) tiledb_force_all_deps="ON";; @@ -155,6 +158,7 @@ done IFS=',' read -ra enables <<< "$enable_multiple" for en in "${enables[@]}"; do case "$en" in + vcpkg) tiledb_vcpkg="ON";; assertions) tiledb_assertions="ON";; debug) build_type="Debug";; release-symbols) build_type="RelWithDebInfo";; @@ -211,6 +215,7 @@ ${cmake} -DCMAKE_BUILD_TYPE=${build_type} \ -DCMAKE_C_COMPILER="${c_compiler}" \ -DCMAKE_CXX_COMPILER="${cxx_compiler}" \ -DCMAKE_PREFIX_PATH="${dependency_dir}" \ + -DTILEDB_VCPKG=${tiledb_vcpkg} \ -DTILEDB_ASSERTIONS=${tiledb_assertions} \ -DTILEDB_VERBOSE=${tiledb_verbose} \ -DTILEDB_HDFS=${tiledb_hdfs} \ diff --git a/cmake/Modules/FindAWSSDK_EP.cmake b/cmake/Modules/FindAWSSDK_EP.cmake index 0870a88436c..a43696db89a 100644 --- a/cmake/Modules/FindAWSSDK_EP.cmake +++ b/cmake/Modules/FindAWSSDK_EP.cmake @@ -31,6 +31,43 @@ # Include some common helper functions. include(TileDBCommon) +if(TILEDB_VCPKG) + find_package(AWSSDK QUIET CONFIG COMPONENTS core s3 REQUIRED) + set(AWS_SERVICES s3) + AWSSDK_DETERMINE_LIBS_TO_LINK(AWS_SERVICES AWS_LINKED_LIBS) + list(APPEND AWS_LINKED_LIBS aws-c-cal + aws-c-io + aws-cpp-sdk-identity-management + aws-cpp-sdk-sts) + + foreach (LIB ${AWS_LINKED_LIBS}) + if (NOT ${LIB} MATCHES "aws-*") + continue() + endif() + + find_library("AWS_FOUND_${LIB}" + NAMES ${LIB} + PATHS ${AWSSDK_LIB_DIR} + NO_DEFAULT_PATH + ) + message(STATUS "Found AWS lib: ${LIB} (${AWS_FOUND_${LIB}})") + if (NOT TARGET AWSSDK::${LIB}) + add_library(AWSSDK::${LIB} UNKNOWN IMPORTED) + set_target_properties(AWSSDK::${LIB} PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${AWSSDK_INCLUDE_DIR}" + IMPORTED_LOCATION "${AWS_FOUND_${LIB}}" + ) + endif() + endforeach () + + # Add missing link directives here rather than adding + # conditional logic in tiledb/CMakeLists.txt + target_link_libraries(AWSSDK::aws-cpp-sdk-s3 INTERFACE AWSSDK::aws-c-cal) + target_link_libraries(AWSSDK::aws-cpp-sdk-s3 INTERFACE AWSSDK::aws-c-io) + + return() +endif() + ##----------------------------------- # early WIN32 audit of path length for aws sdk build where # insufficient available path length causes sdk build failure. diff --git a/cmake/Modules/FindMagic_EP.cmake b/cmake/Modules/FindMagic_EP.cmake index 5bde8a1c9b0..3b5fba8ce7d 100644 --- a/cmake/Modules/FindMagic_EP.cmake +++ b/cmake/Modules/FindMagic_EP.cmake @@ -34,6 +34,40 @@ # Include some common helper functions. include(TileDBCommon) +if(TILEDB_VCPKG) + find_path(libmagic_INCLUDE_DIR NAMES magic.h) + find_library(libmagic_LIBRARIES magic) + find_file(libmagic_DICTIONARY magic.mgc + PATH_SUFFIXES share/libmagic/misc + ) + + include(FindPackageHandleStandardArgs) + FIND_PACKAGE_HANDLE_STANDARD_ARGS(libmagic + REQUIRED_VARS + libmagic_INCLUDE_DIR + libmagic_LIBRARIES + libmagic_DICTIONARY + ) + + if(NOT libmagic_FOUND) + message(FATAL "Error finding libmagic") + endif() + + add_library(libmagic UNKNOWN IMPORTED) + set_target_properties(libmagic PROPERTIES + IMPORTED_LOCATION "${libmagic_LIBRARIES}" + INTERFACE_INCLUDE_DIRECTORIES "${libmagic_INCLUDE_DIR}" + ) + + # Some GitHub builders were finding a system installed liblzma when + # building the libmagic port. Rather than fight the issue we just force + # liblzma support everywhere. + find_package(liblzma CONFIG REQUIRED) + target_link_libraries(libmagic INTERFACE liblzma::liblzma) + + return() +endif() + # Search the path set during the superbuild for the EP. set(LIBMAGIC_PATHS ${TILEDB_EP_INSTALL_PREFIX}) diff --git a/cmake/Modules/FindOpenSSL_EP.cmake b/cmake/Modules/FindOpenSSL_EP.cmake index 8bdd315c620..99c13ec84b2 100644 --- a/cmake/Modules/FindOpenSSL_EP.cmake +++ b/cmake/Modules/FindOpenSSL_EP.cmake @@ -124,7 +124,9 @@ if (NOT OPENSSL_FOUND AND TILEDB_SUPERBUILD) endif() if (OPENSSL_FOUND) - message(STATUS "Found OpenSSL: ${OPENSSL_SSL_LIBRARY} -- OpenSSL crypto: ${OPENSSL_CRYPTO_LIBRARY} -- root: ${OPENSSL_ROOT_DIR}") + message(STATUS "Found OpenSSL: ${OPENSSL_SSL_LIBRARY}") + message(STATUS "Found OpenSSL crypto: ${OPENSSL_CRYPTO_LIBRARY}") + message(STATUS "OpenSSL Root: ${OPENSSL_ROOT_DIR}") if (DEFINED OPENSSL_INCLUDE_DIR AND "${OPENSSL_INCLUDE_DIR}" MATCHES "${HOMEBREW_BASE}.*") # define TILEDB_OPENSSL_DIR so we can ensure curl links the homebrew version diff --git a/cmake/Modules/FindWebp_EP.cmake b/cmake/Modules/FindWebp_EP.cmake index 0598e386ec8..e92a4f334e8 100644 --- a/cmake/Modules/FindWebp_EP.cmake +++ b/cmake/Modules/FindWebp_EP.cmake @@ -27,7 +27,13 @@ # Include some common helper functions. include(TileDBCommon) -if (TILEDB_WEBP_EP_BUILT) +if (TILEDB_VCPKG) + find_package(Threads REQUIRED) + find_package(WebP REQUIRED) + return() +endif() + +if(TILEDB_WEBP_EP_BUILT) find_package(WebP REQUIRED PATHS ${TILEDB_EP_INSTALL_PREFIX} ${TILEDB_DEPS_NO_DEFAULT_PATH}) endif() @@ -43,7 +49,7 @@ if(NOT TILEDB_WEBP_EP_BUILT) #GIT_TAG "release-1.?.?" # after 'static' addition in some release # from branch 'main' history as the 'static' support added apr 12 2022 # at implementation time is not yet in release branch/tag. - GIT_TAG "a19a25bb03757d5bb14f8d9755ab39f06d0ae5ef" + GIT_TAG "a19a25bb03757d5bb14f8d9755ab39f06d0ae5ef" GIT_SUBMODULES_RECURSE TRUE UPDATE_COMMAND "" CMAKE_ARGS diff --git a/cmake/Options/BuildOptions.cmake b/cmake/Options/BuildOptions.cmake index 23e6386d0e6..0b586e14f79 100644 --- a/cmake/Options/BuildOptions.cmake +++ b/cmake/Options/BuildOptions.cmake @@ -6,6 +6,7 @@ # in TileDB-Superbuild.cmake. option(TILEDB_SUPERBUILD "If true, perform a superbuild (builds all missing dependencies)." ON) +option(TILEDB_VCPKG "If true, use vcpkg to download and build dependencies." OFF) option(TILEDB_FORCE_ALL_DEPS "If true, force superbuild to download and build all dependencies, even those installed on the system." OFF) option(TILEDB_REMOVE_DEPRECATIONS "If true, do not build deprecated APIs." OFF) option(TILEDB_VERBOSE "Prints TileDB errors with verbosity" OFF) diff --git a/cmake/Options/TileDBToolchain.cmake b/cmake/Options/TileDBToolchain.cmake index f4086b120e4..57d7e518cae 100644 --- a/cmake/Options/TileDBToolchain.cmake +++ b/cmake/Options/TileDBToolchain.cmake @@ -3,7 +3,17 @@ ############################################################ # Only enable vcpkg on GCS builds for now -if (NOT TILEDB_GCS) +if (NOT TILEDB_VCPKG AND NOT TILEDB_GCS) + return() +endif() + +# For testing we're using --enable-gcs +if(TILEDB_GCS AND NOT TILEDB_VCPKG) + set(TILEDB_VCPKG ON) +endif() + +# We've already run vcpkg by the time the super build is finished +if (NOT TILEDB_SUPERBUILD) return() endif() @@ -19,4 +29,20 @@ else() endif() set(VCPKG_INSTALL_OPTIONS "--no-print-usage") -list(APPEND VCPKG_MANIFEST_FEATURES "gcs") + +macro(tiledb_vcpkg_enable_if tiledb_feature vcpkg_feature) + if(${tiledb_feature}) + list(APPEND VCPKG_MANIFEST_FEATURES ${vcpkg_feature}) + endif() +endmacro() + +tiledb_vcpkg_enable_if(TILEDB_ABSEIL "abseil") +tiledb_vcpkg_enable_if(TILEDB_AZURE "azure") +tiledb_vcpkg_enable_if(TILEDB_GCS "gcs") +tiledb_vcpkg_enable_if(TILEDB_SERIALIZATION "serialization") +tiledb_vcpkg_enable_if(TILEDB_S3 "s3") +tiledb_vcpkg_enable_if(TILEDB_TESTS "tests") +tiledb_vcpkg_enable_if(TILEDB_TOOLS "tools") +tiledb_vcpkg_enable_if(TILEDB_WEBP "webp") + +message("Using vcpkg features: ${VCPKG_MANIFEST_FEATURES}") diff --git a/cmake/TileDB-Superbuild.cmake b/cmake/TileDB-Superbuild.cmake index cd56cc5c7de..0b5dc1ba42c 100644 --- a/cmake/TileDB-Superbuild.cmake +++ b/cmake/TileDB-Superbuild.cmake @@ -29,6 +29,7 @@ set(INHERITED_CMAKE_ARGS -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD} -DCOMPILER_SUPPORTS_AVX2=${COMPILER_SUPPORTS_AVX2} + -DTILEDB_VCPKG=${TILEDB_VCPKG} -DTILEDB_VERBOSE=${TILEDB_VERBOSE} -DTILEDB_ASSERTIONS=${TILEDB_ASSERTIONS} -DTILEDB_S3=${TILEDB_S3} diff --git a/test/external/src/absl_library_targets.cmake b/test/external/src/absl_library_targets.cmake index d0bccef86fd..5115dd5127f 100644 --- a/test/external/src/absl_library_targets.cmake +++ b/test/external/src/absl_library_targets.cmake @@ -31,5 +31,141 @@ # update list as needed if changes in abseil require set(TILEDB_ABSL_LIBRARY_TARGETS - absl::atomic_hook absl::errno_saver absl::log_severity absl::raw_logging_internal absl::spinlock_wait absl::config absl::dynamic_annotations absl::core_headers absl::malloc_internal absl::base_internal absl::base absl::throw_delegate absl::pretty_function absl::endian absl::scoped_set_env absl::strerror absl::fast_type_id absl::algorithm absl::algorithm_container absl::cleanup_internal absl::cleanup absl::btree absl::compressed_tuple absl::fixed_array absl::inlined_vector_internal absl::inlined_vector absl::counting_allocator absl::flat_hash_map absl::flat_hash_set absl::node_hash_map absl::node_hash_set absl::container_memory absl::hash_function_defaults absl::hash_policy_traits absl::hashtablez_sampler absl::hashtable_debug absl::hashtable_debug_hooks absl::have_sse absl::node_hash_policy absl::raw_hash_map absl::container_common absl::raw_hash_set absl::layout absl::stacktrace absl::symbolize absl::examine_stack absl::failure_signal_handler absl::debugging_internal absl::demangle_internal absl::leak_check absl::leak_check_disable absl::debugging absl::flags_path_util absl::flags_program_name absl::flags_config absl::flags_marshalling absl::flags_commandlineflag_internal absl::flags_commandlineflag absl::flags_private_handle_accessor absl::flags_reflection absl::flags_internal absl::flags absl::flags_usage_internal absl::flags_usage absl::flags_parse absl::bind_front absl::function_ref absl::hash absl::city absl::low_level_hash absl::memory absl::type_traits absl::meta absl::bits absl::int128 absl::numeric absl::numeric_representation absl::sample_recorder absl::exponential_biased absl::periodic_sampler absl::random_random absl::random_bit_gen_ref absl::random_internal_mock_helpers absl::random_distributions absl::random_seed_gen_exception absl::random_seed_sequences absl::random_internal_traits absl::random_internal_distribution_caller absl::random_internal_fast_uniform_bits absl::random_internal_seed_material absl::random_internal_pool_urbg absl::random_internal_salted_seed_seq absl::random_internal_iostream_state_saver absl::random_internal_generate_real absl::random_internal_wide_multiply absl::random_internal_fastmath absl::random_internal_nonsecure_base absl::random_internal_pcg_engine absl::random_internal_randen_engine absl::random_internal_platform absl::random_internal_randen absl::random_internal_randen_slow absl::random_internal_randen_hwaes absl::random_internal_randen_hwaes_impl absl::random_internal_distribution_test_util absl::random_internal_uniform_helper absl::status absl::statusor absl::strings absl::strings_internal absl::str_format absl::str_format_internal absl::cord_internal absl::cordz_update_tracker absl::cordz_functions absl::cordz_statistics absl::cordz_handle absl::cordz_info absl::cordz_sample_token absl::cordz_update_scope absl::cord absl::graphcycles_internal absl::kernel_timeout_internal absl::synchronization absl::time absl::civil_time absl::time_zone absl::any absl::bad_any_cast absl::bad_any_cast_impl absl::span absl::optional absl::bad_optional_access absl::bad_variant_access absl::variant absl::compare absl::utility - ) + absl::algorithm + absl::algorithm_container + absl::any + absl::atomic_hook + absl::bad_any_cast + absl::bad_any_cast_impl + absl::bad_optional_access + absl::bad_variant_access + absl::base + absl::base_internal + absl::bind_front + absl::bits + absl::btree + absl::city + absl::civil_time + absl::cleanup + absl::cleanup_internal + absl::compare + absl::compressed_tuple + absl::config + absl::container_common + absl::container_memory + absl::cord + absl::cord_internal + absl::cordz_functions + absl::cordz_handle + absl::cordz_info + absl::cordz_sample_token + absl::cordz_statistics + absl::cordz_update_scope + absl::cordz_update_tracker + absl::core_headers + absl::counting_allocator + absl::debugging + absl::debugging_internal + absl::demangle_internal + absl::dynamic_annotations + absl::endian + absl::errno_saver + absl::examine_stack + absl::exponential_biased + absl::failure_signal_handler + absl::fast_type_id + absl::fixed_array + absl::flags + absl::flags_commandlineflag + absl::flags_commandlineflag_internal + absl::flags_config + absl::flags_internal + absl::flags_marshalling + absl::flags_parse + absl::flags_path_util + absl::flags_private_handle_accessor + absl::flags_program_name + absl::flags_reflection + absl::flags_usage + absl::flags_usage_internal + absl::flat_hash_map + absl::flat_hash_set + absl::function_ref + absl::graphcycles_internal + absl::hash + absl::hash_function_defaults + absl::hash_policy_traits + absl::hashtable_debug + absl::hashtable_debug_hooks + absl::hashtablez_sampler + absl::have_sse + absl::inlined_vector + absl::inlined_vector_internal + absl::int128 + absl::kernel_timeout_internal + absl::layout + absl::leak_check + absl::leak_check_disable + absl::log_severity + absl::low_level_hash + absl::malloc_internal + absl::memory + absl::meta + absl::node_hash_map + absl::node_hash_policy + absl::node_hash_set + absl::numeric + absl::numeric_representation + absl::optional + absl::periodic_sampler + absl::pretty_function + absl::random_bit_gen_ref + absl::random_distributions + absl::random_internal_distribution_caller + absl::random_internal_distribution_test_util + absl::random_internal_fast_uniform_bits + absl::random_internal_fastmath + absl::random_internal_generate_real + absl::random_internal_iostream_state_saver + absl::random_internal_mock_helpers + absl::random_internal_nonsecure_base + absl::random_internal_pcg_engine + absl::random_internal_platform + absl::random_internal_pool_urbg + absl::random_internal_randen + absl::random_internal_randen_engine + absl::random_internal_randen_hwaes + absl::random_internal_randen_hwaes_impl + absl::random_internal_randen_slow + absl::random_internal_salted_seed_seq + absl::random_internal_seed_material + absl::random_internal_traits + absl::random_internal_uniform_helper + absl::random_internal_wide_multiply + absl::random_random + absl::random_seed_gen_exception + absl::random_seed_sequences + absl::raw_hash_map + absl::raw_hash_set + absl::raw_logging_internal + absl::sample_recorder + absl::scoped_set_env + absl::span + absl::spinlock_wait + absl::stacktrace + absl::status + absl::statusor + absl::str_format + absl::str_format_internal + absl::strerror + absl::strings + absl::strings_internal + absl::symbolize + absl::synchronization + absl::throw_delegate + absl::time + absl::time_zone + absl::type_traits + absl::utility + absl::variant +) diff --git a/test/external/src/absl_link_test.cc b/test/external/src/absl_link_test.cc index 6bf3cb601a6..0b7818f82e0 100644 --- a/test/external/src/absl_link_test.cc +++ b/test/external/src/absl_link_test.cc @@ -28,9 +28,9 @@ * * @section DESCRIPTION * - * Tests for linkage to libraries of the targets found to be referenced by + * Tests for linkage to libraries of the targets found to be referenced by * gcssdk v1.22 at the time of creation of absl_link_test.cc. - * + * * Those targets were crudely identified by extracting from the following action: * ep_gcssdk\google>findstr /s absl:: *cmake* cloud\bigquery\CMakeLists.txt: PUBLIC absl::memory absl::strings google_cloud_cpp_grpc_utils @@ -77,23 +77,23 @@ cloud\testing_util\CMakeLists.txt: PUBLIC absl::symbolize absl::failure_s #include #include -#include // absl::str_format -#include // absl::function_ref -#include // absl::fixed_array -#include // absl::flat_hash_map -#include -#include // absl::function_ref -#include // absl::numeric -#include -#include -#include -#include // absl::optional -#include // absl::variant +#include // absl::function_ref +#include // absl::fixed_array +#include // absl::flat_hash_map +#include +#include // absl::function_ref +#include // absl::numeric +#include +#include // absl::str_format +#include +#include +#include // absl::optional +#include // absl::variant #if 0 // absl::symbolize, absl::failure_signal_handler both in testing_util, not // verifying these as we do not build gcs tests. -#include -#include +#include int main() { uint8_t buf[32]; diff --git a/vcpkg-configuration.json b/vcpkg-configuration.json new file mode 100644 index 00000000000..ab5cc21edef --- /dev/null +++ b/vcpkg-configuration.json @@ -0,0 +1,3 @@ +{ + "overlay-ports": ["ports"] +} diff --git a/vcpkg.json b/vcpkg.json index e90de9aa9ca..a226dfdf4df 100644 --- a/vcpkg.json +++ b/vcpkg.json @@ -4,14 +4,36 @@ "builtin-baseline": "f14984af3738e69f197bf0e647a8dca12de92996", "dependencies": [ "bzip2", - "openssl", - "zstd", + "liblzma", + "libmagic", + "lz4", { - "name": "catch2", - "version>=": "3.1.0" - } + "name": "openssl", + "platform": "!windows" + }, + "spdlog", + "zlib", + "zstd" ], "features": { + "abseil": { + "description": "Enables building of abseil library and linkage test", + "dependencies": [ + { + "name": "abseil", + "version>=": "20211102.0" + } + ] + }, + "azure": { + "description": "Support Azure Storage", + "dependencies": [ + { + "name": "azure-storage-cpplite", + "version>=": "0.3.0" + } + ] + }, "gcs": { "description": "Support google cloud storage", "dependencies": [ @@ -22,6 +44,51 @@ "default-features": false } ] + }, + "serialization": { + "description": "Enable TileDB Cloud Support", + "dependencies": [ + { + "name": "capnproto", + "version>=": "0.8.0" + } + ] + }, + "s3": { + "description": "Support AWS S3 Storage", + "dependencies": [ + { + "name": "aws-sdk-cpp", + "version>=": "1.8.84", + "features": ["s3", "identity-management", "sts"], + "default-features": false + } + ] + }, + "tests": { + "description": "Build tiledb tests", + "dependencies": [ + "catch2" + ] + }, + "tools": { + "description": "Build tiledb command line tool", + "dependencies": [ + { + "name": "clipp", + "version>=": "1.2.3" + } + ] + }, + "webp": { + "description": "Support WebP compression", + "dependencies": [ + { + "name": "libwebp", + "version>=": "1.2.3" + } + ] + } } }