From dfec1c07bb78083536cec71955f4ee15b85e4cc6 Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Sat, 16 Nov 2024 15:06:01 +0100 Subject: [PATCH 1/4] define CPPHTTPLIB_ZLIB_SUPPORT required to support gzip content --- src/http_client_extension.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/http_client_extension.cpp b/src/http_client_extension.cpp index dee7518..9fec2cf 100644 --- a/src/http_client_extension.cpp +++ b/src/http_client_extension.cpp @@ -10,6 +10,7 @@ #include #define CPPHTTPLIB_OPENSSL_SUPPORT +#define CPPHTTPLIB_ZLIB_SUPPORT #include "httplib.hpp" #include From c40258c9545aee963ed19c7af2e8533748454dd0 Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Sat, 16 Nov 2024 15:17:58 +0100 Subject: [PATCH 2/4] Update CMakeLists.txt --- CMakeLists.txt | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a6ede33..f5ef350 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,15 +3,18 @@ cmake_minimum_required(VERSION 3.5) # Set extension name here set(TARGET_NAME http_client) -# DuckDB's extension distribution supports vcpkg. As such, dependencies can be added in ./vcpkg.json and then -# used in cmake with find_package. Feel free to remove or replace with other dependencies. -# Note that it should also be removed from vcpkg.json to prevent needlessly installing it.. +# Add ZLIB definition for httplib +add_compile_definitions(CPPHTTPLIB_ZLIB_SUPPORT) + +# Find required packages find_package(OpenSSL REQUIRED) +find_package(ZLIB REQUIRED) set(EXTENSION_NAME ${TARGET_NAME}_extension) set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension) project(${TARGET_NAME}) + include_directories(src/include duckdb/third_party/httplib) set(EXTENSION_SOURCES src/http_client_extension.cpp) @@ -20,23 +23,30 @@ if(MINGW) set(OPENSSL_USE_STATIC_LIBS TRUE) endif() -# Find OpenSSL before building extensions -find_package(OpenSSL REQUIRED) - +# Build extensions build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES}) +# Include directories include_directories(${OPENSSL_INCLUDE_DIR}) -target_link_libraries(${LOADABLE_EXTENSION_NAME} duckdb_mbedtls ${OPENSSL_LIBRARIES}) -target_link_libraries(${EXTENSION_NAME} duckdb_mbedtls ${OPENSSL_LIBRARIES}) +# Common libraries needed for both targets +set(COMMON_LIBS + duckdb_mbedtls + ${OPENSSL_LIBRARIES} + ZLIB::ZLIB +) + +# Windows-specific libraries if(MINGW) set(WIN_LIBS crypt32 ws2_32 wsock32) - find_package(ZLIB) - target_link_libraries(${LOADABLE_EXTENSION_NAME} ZLIB::ZLIB ${WIN_LIBS}) - target_link_libraries(${EXTENSION_NAME} ZLIB::ZLIB ${WIN_LIBS}) + list(APPEND COMMON_LIBS ${WIN_LIBS}) endif() +# Link libraries +target_link_libraries(${LOADABLE_EXTENSION_NAME} ${COMMON_LIBS}) +target_link_libraries(${EXTENSION_NAME} ${COMMON_LIBS}) + install( TARGETS ${EXTENSION_NAME} EXPORT "${DUCKDB_EXPORT_SET}" From fad98fa0fc4e4faaf8c83d50f6ffacc7dde7df46 Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Sat, 16 Nov 2024 15:31:50 +0100 Subject: [PATCH 3/4] Conditional ZLIB support --- CMakeLists.txt | 52 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f5ef350..b2a10ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,12 +3,15 @@ cmake_minimum_required(VERSION 3.5) # Set extension name here set(TARGET_NAME http_client) -# Add ZLIB definition for httplib -add_compile_definitions(CPPHTTPLIB_ZLIB_SUPPORT) +# Make ZLIB support optional with default ON for desktop platforms and OFF for WASM +if(EMSCRIPTEN) + option(USE_ZLIB "Enable ZLIB compression support" OFF) +else() + option(USE_ZLIB "Enable ZLIB compression support" ON) +endif() -# Find required packages +# Find OpenSSL before building extensions find_package(OpenSSL REQUIRED) -find_package(ZLIB REQUIRED) set(EXTENSION_NAME ${TARGET_NAME}_extension) set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension) @@ -20,35 +23,46 @@ include_directories(src/include duckdb/third_party/httplib) set(EXTENSION_SOURCES src/http_client_extension.cpp) if(MINGW) - set(OPENSSL_USE_STATIC_LIBS TRUE) + set(OPENSSL_USE_STATIC_LIBS TRUE) endif() -# Build extensions -build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) -build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES}) - -# Include directories -include_directories(${OPENSSL_INCLUDE_DIR}) - # Common libraries needed for both targets set(COMMON_LIBS duckdb_mbedtls ${OPENSSL_LIBRARIES} - ZLIB::ZLIB ) +# Handle ZLIB support +if(USE_ZLIB) + find_package(ZLIB) + if(ZLIB_FOUND) + add_compile_definitions(CPPHTTPLIB_ZLIB_SUPPORT) + list(APPEND COMMON_LIBS ZLIB::ZLIB) + message(STATUS "Building with ZLIB support") + else() + message(STATUS "ZLIB not found, building without ZLIB support") + endif() +endif() + # Windows-specific libraries if(MINGW) - set(WIN_LIBS crypt32 ws2_32 wsock32) - list(APPEND COMMON_LIBS ${WIN_LIBS}) + set(WIN_LIBS crypt32 ws2_32 wsock32) + list(APPEND COMMON_LIBS ${WIN_LIBS}) endif() +# Build extensions +build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) +build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES}) + +# Include directories +include_directories(${OPENSSL_INCLUDE_DIR}) + # Link libraries target_link_libraries(${LOADABLE_EXTENSION_NAME} ${COMMON_LIBS}) target_link_libraries(${EXTENSION_NAME} ${COMMON_LIBS}) install( - TARGETS ${EXTENSION_NAME} - EXPORT "${DUCKDB_EXPORT_SET}" - LIBRARY DESTINATION "${INSTALL_LIB_DIR}" - ARCHIVE DESTINATION "${INSTALL_LIB_DIR}") + TARGETS ${EXTENSION_NAME} + EXPORT "${DUCKDB_EXPORT_SET}" + LIBRARY DESTINATION "${INSTALL_LIB_DIR}" + ARCHIVE DESTINATION "${INSTALL_LIB_DIR}") From 9971ad8d224fa871e7de29f72bee9b5839ba4cae Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Sat, 16 Nov 2024 15:40:26 +0100 Subject: [PATCH 4/4] Update http_client_extension.cpp --- src/http_client_extension.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/http_client_extension.cpp b/src/http_client_extension.cpp index 9fec2cf..1ddb68a 100644 --- a/src/http_client_extension.cpp +++ b/src/http_client_extension.cpp @@ -9,8 +9,11 @@ #include "duckdb/common/exception/http_exception.hpp" #include -#define CPPHTTPLIB_OPENSSL_SUPPORT +#ifdef USE_ZLIB #define CPPHTTPLIB_ZLIB_SUPPORT +#endif + +#define CPPHTTPLIB_OPENSSL_SUPPORT #include "httplib.hpp" #include