Allow use of vcpkg for handling local dependencies

.github/workflows/ci-linux_mac.yml

@@ -96,7 +96,7 @@ jobs:
         if: ${{ startsWith(matrix.os, 'macos-') == true }}
         run: |
           set -e pipefail
-          brew install pkg-config
+          brew install automake pkg-config
 
       - name: 'Configure libtiledb build'
         id: configure

bootstrap

@@ -34,6 +34,7 @@ Configuration:
     --help                          print this message
     --prefix=PREFIX                 install files in tree rooted at PREFIX
                                     ['"${default_prefix}"']
+    --enable-vcpkg                  use vcpkg for downloading and building dependnecies
     --dependency=DIRs               specify the dependencies at DIRs, separated by colon
                                     ['"${default_dependency}"']
     --force-build-all-deps          force building of all dependencies, even those
@@ -84,6 +85,7 @@ Example:
 
 # Parse arguments
 prefix_dirs="${default_prefix}"
+tiledb_vcpkg="OFF"
 dependency_dir="${default_dependency}"
 sanitizer=""
 build_type="Release"
@@ -114,6 +116,7 @@ while test $# != 0; do
     case "$1" in
     --prefix=*) dir=`arg "$1"`
                 prefix_dirs="$dir";;
+    --enable-vcpkg) tiledb_vcpkg="ON";;
     --dependency=*) dir=`arg "$1"`
                 dependency_dir="$dir";;
     --force-build-all-deps) tiledb_force_all_deps="ON";;
@@ -155,6 +158,7 @@ done
 IFS=',' read -ra enables <<< "$enable_multiple"
 for en in "${enables[@]}"; do
   case "$en" in
+    vcpkg) tiledb_vcpkg="ON";;
     assertions) tiledb_assertions="ON";;
     debug) build_type="Debug";;
     release-symbols) build_type="RelWithDebInfo";;
@@ -211,6 +215,7 @@ ${cmake} -DCMAKE_BUILD_TYPE=${build_type} \
     -DCMAKE_C_COMPILER="${c_compiler}" \
     -DCMAKE_CXX_COMPILER="${cxx_compiler}" \
     -DCMAKE_PREFIX_PATH="${dependency_dir}" \
+    -DTILEDB_VCPKG=${tiledb_vcpkg} \
     -DTILEDB_ASSERTIONS=${tiledb_assertions} \
     -DTILEDB_VERBOSE=${tiledb_verbose} \
     -DTILEDB_HDFS=${tiledb_hdfs} \

cmake/Modules/FindAWSSDK_EP.cmake

@@ -31,6 +31,43 @@
 # Include some common helper functions.
 include(TileDBCommon)
 
+if(TILEDB_VCPKG)
+  find_package(AWSSDK QUIET CONFIG COMPONENTS core s3 REQUIRED)
+  set(AWS_SERVICES s3)
+  AWSSDK_DETERMINE_LIBS_TO_LINK(AWS_SERVICES AWS_LINKED_LIBS)
+  list(APPEND AWS_LINKED_LIBS aws-c-cal
+                              aws-c-io
+                              aws-cpp-sdk-identity-management
+                              aws-cpp-sdk-sts)
+
+  foreach (LIB ${AWS_LINKED_LIBS})
+    if (NOT ${LIB} MATCHES "aws-*")
+      continue()
+    endif()
+
+    find_library("AWS_FOUND_${LIB}"
+      NAMES ${LIB}
+      PATHS ${AWSSDK_LIB_DIR}
+      NO_DEFAULT_PATH
+    )
+    message(STATUS "Found AWS lib: ${LIB} (${AWS_FOUND_${LIB}})")
+    if (NOT TARGET AWSSDK::${LIB})
+      add_library(AWSSDK::${LIB} UNKNOWN IMPORTED)
+      set_target_properties(AWSSDK::${LIB} PROPERTIES
+        INTERFACE_INCLUDE_DIRECTORIES "${AWSSDK_INCLUDE_DIR}"
+        IMPORTED_LOCATION "${AWS_FOUND_${LIB}}"
+      )
+    endif()
+  endforeach ()
+
+  # Add missing link directives here rather than adding
+  # conditional logic in tiledb/CMakeLists.txt
+  target_link_libraries(AWSSDK::aws-cpp-sdk-s3 INTERFACE AWSSDK::aws-c-cal)
+  target_link_libraries(AWSSDK::aws-cpp-sdk-s3 INTERFACE AWSSDK::aws-c-io)
+
+  return()
+endif()
+
 ##-----------------------------------
 # early WIN32 audit of path length for aws sdk build where
 # insufficient available path length causes sdk build failure.

cmake/Modules/FindMagic_EP.cmake

@@ -34,6 +34,40 @@
 # Include some common helper functions.
 include(TileDBCommon)
 
+if(TILEDB_VCPKG)
+  find_path(libmagic_INCLUDE_DIR NAMES magic.h)
+  find_library(libmagic_LIBRARIES magic)
+  find_file(libmagic_DICTIONARY magic.mgc
+    PATH_SUFFIXES share/libmagic/misc
+  )
+
+  include(FindPackageHandleStandardArgs)
+  FIND_PACKAGE_HANDLE_STANDARD_ARGS(libmagic
+    REQUIRED_VARS
+      libmagic_INCLUDE_DIR
+      libmagic_LIBRARIES
+      libmagic_DICTIONARY
+  )
+
+  if(NOT libmagic_FOUND)
+    message(FATAL "Error finding libmagic")
+  endif()
+
+  add_library(libmagic UNKNOWN IMPORTED)
+  set_target_properties(libmagic PROPERTIES
+    IMPORTED_LOCATION "${libmagic_LIBRARIES}"
+    INTERFACE_INCLUDE_DIRECTORIES "${libmagic_INCLUDE_DIR}"
+  )
+
+  # Some GitHub builders were finding a system installed liblzma when
+  # building the libmagic port. Rather than fight the issue we just force
+  # liblzma support everywhere.
+  find_package(liblzma CONFIG REQUIRED)
+  target_link_libraries(libmagic INTERFACE liblzma::liblzma)
+
+  return()
+endif()
+
 # Search the path set during the superbuild for the EP.
 set(LIBMAGIC_PATHS ${TILEDB_EP_INSTALL_PREFIX})
 

cmake/Modules/FindOpenSSL_EP.cmake

@@ -124,7 +124,9 @@ if (NOT OPENSSL_FOUND AND TILEDB_SUPERBUILD)
 endif()
 
 if (OPENSSL_FOUND)
-  message(STATUS "Found OpenSSL: ${OPENSSL_SSL_LIBRARY} -- OpenSSL crypto: ${OPENSSL_CRYPTO_LIBRARY} -- root: ${OPENSSL_ROOT_DIR}")
+  message(STATUS "Found OpenSSL: ${OPENSSL_SSL_LIBRARY}")
+  message(STATUS "Found OpenSSL crypto: ${OPENSSL_CRYPTO_LIBRARY}")
+  message(STATUS "OpenSSL Root: ${OPENSSL_ROOT_DIR}")
 
   if (DEFINED OPENSSL_INCLUDE_DIR AND "${OPENSSL_INCLUDE_DIR}" MATCHES "${HOMEBREW_BASE}.*")
     # define TILEDB_OPENSSL_DIR so we can ensure curl links the homebrew version

cmake/Modules/FindWebp_EP.cmake

@@ -27,7 +27,13 @@
 # Include some common helper functions.
 include(TileDBCommon)
 
-if (TILEDB_WEBP_EP_BUILT)
+if (TILEDB_VCPKG)
+  find_package(Threads REQUIRED)
+  find_package(WebP REQUIRED)
+  return()
+endif()
+
+if(TILEDB_WEBP_EP_BUILT)
   find_package(WebP REQUIRED PATHS ${TILEDB_EP_INSTALL_PREFIX} ${TILEDB_DEPS_NO_DEFAULT_PATH})
 endif()
 
@@ -43,7 +49,7 @@ if(NOT TILEDB_WEBP_EP_BUILT)
       #GIT_TAG "release-1.?.?" # after 'static' addition in some release
       # from branch 'main' history as the 'static' support added apr 12 2022
       # at implementation time is not yet in release branch/tag.
-      GIT_TAG "a19a25bb03757d5bb14f8d9755ab39f06d0ae5ef" 
+      GIT_TAG "a19a25bb03757d5bb14f8d9755ab39f06d0ae5ef"
       GIT_SUBMODULES_RECURSE TRUE
       UPDATE_COMMAND ""
       CMAKE_ARGS

cmake/Options/BuildOptions.cmake

@@ -6,6 +6,7 @@
 # in TileDB-Superbuild.cmake.
 
 option(TILEDB_SUPERBUILD "If true, perform a superbuild (builds all missing dependencies)." ON)
+option(TILEDB_VCPKG "If true, use vcpkg to download and build dependencies." OFF)
 option(TILEDB_FORCE_ALL_DEPS "If true, force superbuild to download and build all dependencies, even those installed on the system." OFF)
 option(TILEDB_REMOVE_DEPRECATIONS "If true, do not build deprecated APIs." OFF)
 option(TILEDB_VERBOSE "Prints TileDB errors with verbosity" OFF)

cmake/Options/TileDBToolchain.cmake

@@ -3,7 +3,17 @@
 ############################################################
 
 # Only enable vcpkg on GCS builds for now
-if (NOT TILEDB_GCS)
+if (NOT TILEDB_VCPKG AND NOT TILEDB_GCS)
+    return()
+endif()
+
+# For testing we're using --enable-gcs
+if(TILEDB_GCS AND NOT TILEDB_VCPKG)
+    set(TILEDB_VCPKG ON)
+endif()
+
+# We've already run vcpkg by the time the super build is finished
+if (NOT TILEDB_SUPERBUILD)
     return()
 endif()
 
@@ -19,4 +29,20 @@ else()
 endif()
 
 set(VCPKG_INSTALL_OPTIONS "--no-print-usage")
-list(APPEND VCPKG_MANIFEST_FEATURES "gcs")
+
+macro(tiledb_vcpkg_enable_if tiledb_feature vcpkg_feature)
+    if(${tiledb_feature})
+        list(APPEND VCPKG_MANIFEST_FEATURES ${vcpkg_feature})
+    endif()
+endmacro()
+
+tiledb_vcpkg_enable_if(TILEDB_ABSEIL "abseil")
+tiledb_vcpkg_enable_if(TILEDB_AZURE "azure")
+tiledb_vcpkg_enable_if(TILEDB_GCS "gcs")
+tiledb_vcpkg_enable_if(TILEDB_SERIALIZATION "serialization")
+tiledb_vcpkg_enable_if(TILEDB_S3 "s3")
+tiledb_vcpkg_enable_if(TILEDB_TESTS "tests")
+tiledb_vcpkg_enable_if(TILEDB_TOOLS "tools")
+tiledb_vcpkg_enable_if(TILEDB_WEBP "webp")
+
+message("Using vcpkg features: ${VCPKG_MANIFEST_FEATURES}")

cmake/TileDB-Superbuild.cmake

@@ -29,6 +29,7 @@ set(INHERITED_CMAKE_ARGS
   -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
   -DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
   -DCOMPILER_SUPPORTS_AVX2=${COMPILER_SUPPORTS_AVX2}
+  -DTILEDB_VCPKG=${TILEDB_VCPKG}
   -DTILEDB_VERBOSE=${TILEDB_VERBOSE}
   -DTILEDB_ASSERTIONS=${TILEDB_ASSERTIONS}
   -DTILEDB_S3=${TILEDB_S3}

ports/abseil/fix-32-bit-arm.patch

@@ -0,0 +1,23 @@
+diff --git a/absl/time/internal/cctz/src/zone_info_source.cc b/absl/time/internal/cctz/src/zone_info_source.cc
+index 7209533..5ab5a59 100644
+--- a/absl/time/internal/cctz/src/zone_info_source.cc
++++ b/absl/time/internal/cctz/src/zone_info_source.cc
+@@ -65,7 +65,7 @@ ZoneInfoSourceFactory zone_info_source_factory __attribute__((weak)) =
+ extern ZoneInfoSourceFactory zone_info_source_factory;
+ extern ZoneInfoSourceFactory default_factory;
+ ZoneInfoSourceFactory default_factory = DefaultFactory;
+-#if defined(_M_IX86)
++#if defined(_M_IX86) || defined(_M_ARM)
+ #pragma comment(                                                                                                         \
+     linker,                                                                                                              \
+     "/alternatename:?zone_info_source_factory@cctz_extension@time_internal@" ABSL_INTERNAL_MANGLED_NS                    \
+@@ -83,8 +83,7 @@ ZoneInfoSourceFactory default_factory = DefaultFactory;
+     "@@U?$default_delete@VZoneInfoSource@cctz@time_internal@" ABSL_INTERNAL_MANGLED_NS                                   \
+     "@@@std@@@std@@ABV?$basic_string@DU?$char_traits@D@std@@V?$allocator@D@2@@2@@Z@" ABSL_INTERNAL_MANGLED_BACKREFERENCE \
+     "@@ZA")
+-#elif defined(_M_IA_64) || defined(_M_AMD64) || defined(_M_ARM) || \
+-    defined(_M_ARM64)
++#elif defined(_M_IA_64) || defined(_M_AMD64) || defined(_M_ARM64)
+ #pragma comment(                                                                                                          \
+     linker,                                                                                                               \
+     "/alternatename:?zone_info_source_factory@cctz_extension@time_internal@" ABSL_INTERNAL_MANGLED_NS                     \

ports/abseil/fix-cxx-standard.patch

@@ -0,0 +1,94 @@
+diff --git a/CMake/abslConfig.cmake.in b/CMake/abslConfig.cmake.in
+index 62d246d..afcec1f 100644
+--- a/CMake/abslConfig.cmake.in
++++ b/CMake/abslConfig.cmake.in
+@@ -6,3 +6,5 @@ find_dependency(Threads)
+ @PACKAGE_INIT@
+
+ include ("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake")
++
++set(ABSL_USE_CXX17 @ABSL_USE_CXX17@)
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 750a475..239977f 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -67,6 +67,23 @@ else()
+   option(ABSL_ENABLE_INSTALL "Enable install rule" ON)
+ endif()
+
++# CXX standard
++option(ABSL_USE_CXX17 "Enable CXX 17 standard" OFF)
++
++if (ABSL_USE_CXX17)
++    set(CMAKE_CXX_STANDARD 17)
++    set(STD_ANY 1)
++    set(STD_OPTIONAL 1)
++    set(STD_STRING_VIEW 1)
++    set(STD_VARIANT 1)
++else()
++    set(CMAKE_CXX_STANDARD 11)
++    set(STD_ANY 0)
++    set(STD_OPTIONAL 0)
++    set(STD_STRING_VIEW 0)
++    set(STD_VARIANT 0)
++endif()
++
+ option(ABSL_PROPAGATE_CXX_STD
+   "Use CMake C++ standard meta features (e.g. cxx_std_11) that propagate to targets that link to Abseil"
+   OFF)  # TODO: Default to ON for CMake 3.8 and greater.
+diff --git a/absl/base/CMakeLists.txt b/absl/base/CMakeLists.txt
+index c7233cb..65a9076 100644
+--- a/absl/base/CMakeLists.txt
++++ b/absl/base/CMakeLists.txt
+@@ -38,6 +38,11 @@ absl_cc_library(
+   COPTS
+     ${ABSL_DEFAULT_COPTS}
+ )
++if (NOT EXISTS ${CMAKE_CURRENT_LIST_DIR}/options.h.in)
++    file(RENAME ${CMAKE_CURRENT_LIST_DIR}/options.h ${CMAKE_CURRENT_LIST_DIR}/options.h.in)
++endif()
++file(REMOVE ${CMAKE_CURRENT_LIST_DIR}/options.h)
++configure_file(${CMAKE_CURRENT_LIST_DIR}/options.h.in ${CMAKE_CURRENT_LIST_DIR}/options.h @ONLY)
+
+ absl_cc_library(
+   NAME
+diff --git a/absl/base/options.h b/absl/base/options.h
+index 56b4e36..0b098ed 100644
+--- a/absl/base/options.h
++++ b/absl/base/options.h
+@@ -100,7 +100,7 @@
+ // User code should not inspect this macro.  To check in the preprocessor if
+ // absl::any is a typedef of std::any, use the feature macro ABSL_USES_STD_ANY.
+
+-#define ABSL_OPTION_USE_STD_ANY 2
++#define ABSL_OPTION_USE_STD_ANY @STD_ANY@
+
+
+ // ABSL_OPTION_USE_STD_OPTIONAL
+@@ -127,7 +127,7 @@
+ // absl::optional is a typedef of std::optional, use the feature macro
+ // ABSL_USES_STD_OPTIONAL.
+
+-#define ABSL_OPTION_USE_STD_OPTIONAL 2
++#define ABSL_OPTION_USE_STD_OPTIONAL @STD_OPTIONAL@
+
+
+ // ABSL_OPTION_USE_STD_STRING_VIEW
+@@ -154,7 +154,7 @@
+ // absl::string_view is a typedef of std::string_view, use the feature macro
+ // ABSL_USES_STD_STRING_VIEW.
+
+-#define ABSL_OPTION_USE_STD_STRING_VIEW 2
++#define ABSL_OPTION_USE_STD_STRING_VIEW @STD_STRING_VIEW@
+
+ // ABSL_OPTION_USE_STD_VARIANT
+ //
+@@ -180,7 +180,7 @@
+ // absl::variant is a typedef of std::variant, use the feature macro
+ // ABSL_USES_STD_VARIANT.
+
+-#define ABSL_OPTION_USE_STD_VARIANT 2
++#define ABSL_OPTION_USE_STD_VARIANT @STD_VARIANT@
+
+
+ // ABSL_OPTION_USE_INLINE_NAMESPACE