Optionally build all dependencies with vcpkg

This adds a `--enable-vcpkg` configuration flag to CMake that allows
developers to build all dependencies with vcpkg.

.github/workflows/ci-linux_mac.yml

@@ -96,7 +96,7 @@ jobs:
         if: ${{ startsWith(matrix.os, 'macos-') == true }}
         run: |
           set -e pipefail
-          brew install pkg-config
+          brew install automake pkg-config
 
       - name: 'Configure libtiledb build'
         id: configure

bootstrap

@@ -34,6 +34,7 @@ Configuration:
     --help                          print this message
     --prefix=PREFIX                 install files in tree rooted at PREFIX
                                     ['"${default_prefix}"']
+    --enable-vcpkg                  use vcpkg for downloading and building dependnecies
     --dependency=DIRs               specify the dependencies at DIRs, separated by colon
                                     ['"${default_dependency}"']
     --force-build-all-deps          force building of all dependencies, even those
@@ -84,6 +85,7 @@ Example:
 
 # Parse arguments
 prefix_dirs="${default_prefix}"
+tiledb_vcpkg="OFF"
 dependency_dir="${default_dependency}"
 sanitizer=""
 build_type="Release"
@@ -114,6 +116,7 @@ while test $# != 0; do
     case "$1" in
     --prefix=*) dir=`arg "$1"`
                 prefix_dirs="$dir";;
+    --enable-vcpkg) tiledb_vcpkg="ON";;
     --dependency=*) dir=`arg "$1"`
                 dependency_dir="$dir";;
     --force-build-all-deps) tiledb_force_all_deps="ON";;
@@ -155,6 +158,7 @@ done
 IFS=',' read -ra enables <<< "$enable_multiple"
 for en in "${enables[@]}"; do
   case "$en" in
+    vcpkg) tiledb_vcpkg="ON";;
     assertions) tiledb_assertions="ON";;
     debug) build_type="Debug";;
     release-symbols) build_type="RelWithDebInfo";;
@@ -211,6 +215,7 @@ ${cmake} -DCMAKE_BUILD_TYPE=${build_type} \
     -DCMAKE_C_COMPILER="${c_compiler}" \
     -DCMAKE_CXX_COMPILER="${cxx_compiler}" \
     -DCMAKE_PREFIX_PATH="${dependency_dir}" \
+    -DTILEDB_VCPKG=${tiledb_vcpkg} \
     -DTILEDB_ASSERTIONS=${tiledb_assertions} \
     -DTILEDB_VERBOSE=${tiledb_verbose} \
     -DTILEDB_HDFS=${tiledb_hdfs} \

cmake/Modules/FindAWSSDK_EP.cmake

@@ -31,6 +31,43 @@
 # Include some common helper functions.
 include(TileDBCommon)
 
+if(TILEDB_VCPKG)
+  find_package(AWSSDK QUIET CONFIG COMPONENTS core s3 REQUIRED)
+  set(AWS_SERVICES s3)
+  AWSSDK_DETERMINE_LIBS_TO_LINK(AWS_SERVICES AWS_LINKED_LIBS)
+  list(APPEND AWS_LINKED_LIBS aws-c-cal
+                              aws-c-io
+                              aws-cpp-sdk-identity-management
+                              aws-cpp-sdk-sts)
+
+  foreach (LIB ${AWS_LINKED_LIBS})
+    if (NOT ${LIB} MATCHES "aws-*")
+      continue()
+    endif()
+
+    find_library("AWS_FOUND_${LIB}"
+      NAMES ${LIB}
+      PATHS ${AWSSDK_LIB_DIR}
+      NO_DEFAULT_PATH
+    )
+    message(STATUS "Found AWS lib: ${LIB} (${AWS_FOUND_${LIB}})")
+    if (NOT TARGET AWSSDK::${LIB})
+      add_library(AWSSDK::${LIB} UNKNOWN IMPORTED)
+      set_target_properties(AWSSDK::${LIB} PROPERTIES
+        INTERFACE_INCLUDE_DIRECTORIES "${AWSSDK_INCLUDE_DIR}"
+        IMPORTED_LOCATION "${AWS_FOUND_${LIB}}"
+      )
+    endif()
+  endforeach ()
+
+  # Add missing link directives here rather than adding
+  # conditional logic in tiledb/CMakeLists.txt
+  target_link_libraries(AWSSDK::aws-cpp-sdk-s3 INTERFACE AWSSDK::aws-c-cal)
+  target_link_libraries(AWSSDK::aws-cpp-sdk-s3 INTERFACE AWSSDK::aws-c-io)
+
+  return()
+endif()
+
 ##-----------------------------------
 # early WIN32 audit of path length for aws sdk build where
 # insufficient available path length causes sdk build failure.

cmake/Modules/FindMagic_EP.cmake

@@ -34,6 +34,40 @@
 # Include some common helper functions.
 include(TileDBCommon)
 
+if(TILEDB_VCPKG)
+  find_path(libmagic_INCLUDE_DIR NAMES magic.h)
+  find_library(libmagic_LIBRARIES magic)
+  find_file(libmagic_DICTIONARY magic.mgc
+    PATH_SUFFIXES share/libmagic/misc
+  )
+
+  include(FindPackageHandleStandardArgs)
+  FIND_PACKAGE_HANDLE_STANDARD_ARGS(libmagic
+    REQUIRED_VARS
+      libmagic_INCLUDE_DIR
+      libmagic_LIBRARIES
+      libmagic_DICTIONARY
+  )
+
+  if(NOT libmagic_FOUND)
+    message(FATAL "Error finding libmagic")
+  endif()
+
+  add_library(libmagic UNKNOWN IMPORTED)
+  set_target_properties(libmagic PROPERTIES
+    IMPORTED_LOCATION "${libmagic_LIBRARIES}"
+    INTERFACE_INCLUDE_DIRECTORIES "${libmagic_INCLUDE_DIR}"
+  )
+
+  # Some GitHub builders were finding a system installed liblzma when
+  # building the libmagic port. Rather than fight the issue we just force
+  # liblzma support everywhere.
+  find_package(liblzma CONFIG REQUIRED)
+  target_link_libraries(libmagic INTERFACE liblzma::liblzma)
+
+  return()
+endif()
+
 # Search the path set during the superbuild for the EP.
 set(LIBMAGIC_PATHS ${TILEDB_EP_INSTALL_PREFIX})
 

cmake/Modules/FindOpenSSL_EP.cmake

@@ -124,7 +124,9 @@ if (NOT OPENSSL_FOUND AND TILEDB_SUPERBUILD)
 endif()
 
 if (OPENSSL_FOUND)
-  message(STATUS "Found OpenSSL: ${OPENSSL_SSL_LIBRARY} -- OpenSSL crypto: ${OPENSSL_CRYPTO_LIBRARY} -- root: ${OPENSSL_ROOT_DIR}")
+  message(STATUS "Found OpenSSL: ${OPENSSL_SSL_LIBRARY}")
+  message(STATUS "Found OpenSSL crypto: ${OPENSSL_CRYPTO_LIBRARY}")
+  message(STATUS "OpenSSL Root: ${OPENSSL_ROOT_DIR}")
 
   if (DEFINED OPENSSL_INCLUDE_DIR AND "${OPENSSL_INCLUDE_DIR}" MATCHES "${HOMEBREW_BASE}.*")
     # define TILEDB_OPENSSL_DIR so we can ensure curl links the homebrew version

cmake/Modules/FindWebp_EP.cmake

@@ -27,7 +27,13 @@
 # Include some common helper functions.
 include(TileDBCommon)
 
-if (TILEDB_WEBP_EP_BUILT)
+if (TILEDB_VCPKG)
+  find_package(Threads REQUIRED)
+  find_package(WebP REQUIRED)
+  return()
+endif()
+
+if(TILEDB_WEBP_EP_BUILT)
   find_package(WebP REQUIRED PATHS ${TILEDB_EP_INSTALL_PREFIX} ${TILEDB_DEPS_NO_DEFAULT_PATH})
 endif()
 
@@ -43,7 +49,7 @@ if(NOT TILEDB_WEBP_EP_BUILT)
       #GIT_TAG "release-1.?.?" # after 'static' addition in some release
       # from branch 'main' history as the 'static' support added apr 12 2022
       # at implementation time is not yet in release branch/tag.
-      GIT_TAG "a19a25bb03757d5bb14f8d9755ab39f06d0ae5ef" 
+      GIT_TAG "a19a25bb03757d5bb14f8d9755ab39f06d0ae5ef"
       GIT_SUBMODULES_RECURSE TRUE
       UPDATE_COMMAND ""
       CMAKE_ARGS

cmake/Options/BuildOptions.cmake

@@ -6,6 +6,7 @@
 # in TileDB-Superbuild.cmake.
 
 option(TILEDB_SUPERBUILD "If true, perform a superbuild (builds all missing dependencies)." ON)
+option(TILEDB_VCPKG "If true, use vcpkg to download and build dependencies." OFF)
 option(TILEDB_FORCE_ALL_DEPS "If true, force superbuild to download and build all dependencies, even those installed on the system." OFF)
 option(TILEDB_REMOVE_DEPRECATIONS "If true, do not build deprecated APIs." OFF)
 option(TILEDB_VERBOSE "Prints TileDB errors with verbosity" OFF)

cmake/Options/TileDBToolchain.cmake

@@ -3,7 +3,17 @@
 ############################################################
 
 # Only enable vcpkg on GCS builds for now
-if (NOT TILEDB_GCS)
+if (NOT TILEDB_VCPKG AND NOT TILEDB_GCS)
+    return()
+endif()
+
+# For testing we're using --enable-gcs
+if(TILEDB_GCS AND NOT TILEDB_VCPKG)
+    set(TILEDB_VCPKG ON)
+endif()
+
+# We've already run vcpkg by the time the super build is finished
+if (NOT TILEDB_SUPERBUILD)
     return()
 endif()
 
@@ -19,4 +29,20 @@ else()
 endif()
 
 set(VCPKG_INSTALL_OPTIONS "--no-print-usage")
-list(APPEND VCPKG_MANIFEST_FEATURES "gcs")
+
+macro(tiledb_vcpkg_enable_if tiledb_feature vcpkg_feature)
+    if(${tiledb_feature})
+        list(APPEND VCPKG_MANIFEST_FEATURES ${vcpkg_feature})
+    endif()
+endmacro()
+
+tiledb_vcpkg_enable_if(TILEDB_ABSEIL "abseil")
+tiledb_vcpkg_enable_if(TILEDB_AZURE "azure")
+tiledb_vcpkg_enable_if(TILEDB_GCS "gcs")
+tiledb_vcpkg_enable_if(TILEDB_SERIALIZATION "serialization")
+tiledb_vcpkg_enable_if(TILEDB_S3 "s3")
+tiledb_vcpkg_enable_if(TILEDB_TESTS "tests")
+tiledb_vcpkg_enable_if(TILEDB_TOOLS "tools")
+tiledb_vcpkg_enable_if(TILEDB_WEBP "webp")
+
+message("Using vcpkg features: ${VCPKG_MANIFEST_FEATURES}")

cmake/TileDB-Superbuild.cmake

@@ -29,6 +29,7 @@ set(INHERITED_CMAKE_ARGS
   -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
   -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
   -DCOMPILER_SUPPORTS_AVX2=${COMPILER_SUPPORTS_AVX2}
+  -DTILEDB_VCPKG=${TILEDB_VCPKG}
   -DTILEDB_VERBOSE=${TILEDB_VERBOSE}
   -DTILEDB_ASSERTIONS=${TILEDB_ASSERTIONS}
   -DTILEDB_S3=${TILEDB_S3}

test/external/src/absl_library_targets.cmake

@@ -31,5 +31,141 @@
 # update list as needed if changes in abseil require
 
 set(TILEDB_ABSL_LIBRARY_TARGETS
-  absl::atomic_hook absl::errno_saver absl::log_severity absl::raw_logging_internal absl::spinlock_wait absl::config absl::dynamic_annotations absl::core_headers absl::malloc_internal absl::base_internal absl::base absl::throw_delegate absl::pretty_function absl::endian absl::scoped_set_env absl::strerror absl::fast_type_id absl::algorithm absl::algorithm_container absl::cleanup_internal absl::cleanup absl::btree absl::compressed_tuple absl::fixed_array absl::inlined_vector_internal absl::inlined_vector absl::counting_allocator absl::flat_hash_map absl::flat_hash_set absl::node_hash_map absl::node_hash_set absl::container_memory absl::hash_function_defaults absl::hash_policy_traits absl::hashtablez_sampler absl::hashtable_debug absl::hashtable_debug_hooks absl::have_sse absl::node_hash_policy absl::raw_hash_map absl::container_common absl::raw_hash_set absl::layout absl::stacktrace absl::symbolize absl::examine_stack absl::failure_signal_handler absl::debugging_internal absl::demangle_internal absl::leak_check absl::leak_check_disable absl::debugging absl::flags_path_util absl::flags_program_name absl::flags_config absl::flags_marshalling absl::flags_commandlineflag_internal absl::flags_commandlineflag absl::flags_private_handle_accessor absl::flags_reflection absl::flags_internal absl::flags absl::flags_usage_internal absl::flags_usage absl::flags_parse absl::bind_front absl::function_ref absl::hash absl::city absl::low_level_hash absl::memory absl::type_traits absl::meta absl::bits absl::int128 absl::numeric absl::numeric_representation absl::sample_recorder absl::exponential_biased absl::periodic_sampler absl::random_random absl::random_bit_gen_ref absl::random_internal_mock_helpers absl::random_distributions absl::random_seed_gen_exception absl::random_seed_sequences absl::random_internal_traits absl::random_internal_distribution_caller absl::random_internal_fast_uniform_bits absl::random_internal_seed_material absl::random_internal_pool_urbg absl::random_internal_salted_seed_seq absl::random_internal_iostream_state_saver absl::random_internal_generate_real absl::random_internal_wide_multiply absl::random_internal_fastmath absl::random_internal_nonsecure_base absl::random_internal_pcg_engine absl::random_internal_randen_engine absl::random_internal_platform absl::random_internal_randen absl::random_internal_randen_slow absl::random_internal_randen_hwaes absl::random_internal_randen_hwaes_impl absl::random_internal_distribution_test_util absl::random_internal_uniform_helper absl::status absl::statusor absl::strings absl::strings_internal absl::str_format absl::str_format_internal absl::cord_internal absl::cordz_update_tracker absl::cordz_functions absl::cordz_statistics absl::cordz_handle absl::cordz_info absl::cordz_sample_token absl::cordz_update_scope absl::cord absl::graphcycles_internal absl::kernel_timeout_internal absl::synchronization absl::time absl::civil_time absl::time_zone absl::any absl::bad_any_cast absl::bad_any_cast_impl absl::span absl::optional absl::bad_optional_access absl::bad_variant_access absl::variant absl::compare absl::utility
-  )
+  absl::algorithm
+  absl::algorithm_container
+  absl::any
+  absl::atomic_hook
+  absl::bad_any_cast
+  absl::bad_any_cast_impl
+  absl::bad_optional_access
+  absl::bad_variant_access
+  absl::base
+  absl::base_internal
+  absl::bind_front
+  absl::bits
+  absl::btree
+  absl::city
+  absl::civil_time
+  absl::cleanup
+  absl::cleanup_internal
+  absl::compare
+  absl::compressed_tuple
+  absl::config
+  absl::container_common
+  absl::container_memory
+  absl::cord
+  absl::cord_internal
+  absl::cordz_functions
+  absl::cordz_handle
+  absl::cordz_info
+  absl::cordz_sample_token
+  absl::cordz_statistics
+  absl::cordz_update_scope
+  absl::cordz_update_tracker
+  absl::core_headers
+  absl::counting_allocator
+  absl::debugging
+  absl::debugging_internal
+  absl::demangle_internal
+  absl::dynamic_annotations
+  absl::endian
+  absl::errno_saver
+  absl::examine_stack
+  absl::exponential_biased
+  absl::failure_signal_handler
+  absl::fast_type_id
+  absl::fixed_array
+  absl::flags
+  absl::flags_commandlineflag
+  absl::flags_commandlineflag_internal
+  absl::flags_config
+  absl::flags_internal
+  absl::flags_marshalling
+  absl::flags_parse
+  absl::flags_path_util
+  absl::flags_private_handle_accessor
+  absl::flags_program_name
+  absl::flags_reflection
+  absl::flags_usage
+  absl::flags_usage_internal
+  absl::flat_hash_map
+  absl::flat_hash_set
+  absl::function_ref
+  absl::graphcycles_internal
+  absl::hash
+  absl::hash_function_defaults
+  absl::hash_policy_traits
+  absl::hashtable_debug
+  absl::hashtable_debug_hooks
+  absl::hashtablez_sampler
+  absl::have_sse
+  absl::inlined_vector
+  absl::inlined_vector_internal
+  absl::int128
+  absl::kernel_timeout_internal
+  absl::layout
+  absl::leak_check
+  absl::leak_check_disable
+  absl::log_severity
+  absl::low_level_hash
+  absl::malloc_internal
+  absl::memory
+  absl::meta
+  absl::node_hash_map
+  absl::node_hash_policy
+  absl::node_hash_set
+  absl::numeric
+  absl::numeric_representation
+  absl::optional
+  absl::periodic_sampler
+  absl::pretty_function
+  absl::random_bit_gen_ref
+  absl::random_distributions
+  absl::random_internal_distribution_caller
+  absl::random_internal_distribution_test_util
+  absl::random_internal_fast_uniform_bits
+  absl::random_internal_fastmath
+  absl::random_internal_generate_real
+  absl::random_internal_iostream_state_saver
+  absl::random_internal_mock_helpers
+  absl::random_internal_nonsecure_base
+  absl::random_internal_pcg_engine
+  absl::random_internal_platform
+  absl::random_internal_pool_urbg
+  absl::random_internal_randen
+  absl::random_internal_randen_engine
+  absl::random_internal_randen_hwaes
+  absl::random_internal_randen_hwaes_impl
+  absl::random_internal_randen_slow
+  absl::random_internal_salted_seed_seq
+  absl::random_internal_seed_material
+  absl::random_internal_traits
+  absl::random_internal_uniform_helper
+  absl::random_internal_wide_multiply
+  absl::random_random
+  absl::random_seed_gen_exception
+  absl::random_seed_sequences
+  absl::raw_hash_map
+  absl::raw_hash_set
+  absl::raw_logging_internal
+  absl::sample_recorder
+  absl::scoped_set_env
+  absl::span
+  absl::spinlock_wait
+  absl::stacktrace
+  absl::status
+  absl::statusor
+  absl::str_format
+  absl::str_format_internal
+  absl::strerror
+  absl::strings
+  absl::strings_internal
+  absl::symbolize
+  absl::synchronization
+  absl::throw_delegate
+  absl::time
+  absl::time_zone
+  absl::type_traits
+  absl::utility
+  absl::variant
+)