From d77d50909f16c05fc5d0551f7053e2a4c670dd9c Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Mon, 13 Oct 2025 12:34:31 +0800 Subject: [PATCH 01/41] chore: update contrib/pybind11 --- CMakeLists.txt | 9 +- PreLoad.cmake | 2 +- chdb/build.sh | 139 +++++++++--------- chdb/vars.sh | 3 +- cmake/cpu_features.cmake | 28 ++-- .../include/jemalloc/jemalloc_defs.h | 4 +- .../internal/jemalloc_internal_defs.h.in | 7 +- contrib/pybind11 | 2 +- contrib/pybind11-cmake/CMakeLists.txt | 57 ++++++- setup.py | 3 +- src/Access/Common/AccessFlags.cpp | 11 +- src/Common/AllocationInterceptors.cpp | 94 +++++++++++- src/Common/memory.h | 27 ++++ 13 files changed, 282 insertions(+), 104 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 099ff907e9a..37c8bc799ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -163,6 +163,13 @@ if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWIT set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -fuse-ctor-homing") endif() +# Fix TLS model for MUSL to avoid initial-exec TLS relocation errors +if (USE_MUSL) + add_definitions(-DUSE_MUSL=1 -D__MUSL__=1) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ftls-model=global-dynamic") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ftls-model=global-dynamic") +endif() + option(ENABLE_TESTS "Provide unit_test_dbms target with Google.Test unit tests" ON) option(ENABLE_EXAMPLES "Build all example programs in 'examples' subdirectories" OFF) option(ENABLE_BENCHMARKS "Build all benchmark programs in 'benchmarks' subdirectories" OFF) @@ -396,7 +403,7 @@ option(ENABLE_LIBRARIES "Enable all external libraries by default" ON) # Increase stack size on Musl. We need big stack for our recursive-descend parser. if (USE_MUSL) - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-stack_size,2097152") + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-z,stack-size=2097152") endif () include(cmake/dbms_glob_sources.cmake) diff --git a/PreLoad.cmake b/PreLoad.cmake index 2124b7c0e9b..0ac0a8b14be 100644 --- a/PreLoad.cmake +++ b/PreLoad.cmake @@ -91,7 +91,7 @@ if (OS MATCHES "Linux" # - compile musl with debug and -fasynchronous-unwind-tables # # But none of this changes anything so far. - set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-x86_64.cmake" CACHE INTERNAL "") + # set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-x86_64.cmake" CACHE INTERNAL "") elseif (ARCH MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)") set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-aarch64.cmake" CACHE INTERNAL "") elseif (ARCH MATCHES "^(ppc64le.*|PPC64LE.*)") diff --git a/chdb/build.sh b/chdb/build.sh index f31e904e964..7cf58634308 100755 --- a/chdb/build.sh +++ b/chdb/build.sh @@ -3,7 +3,7 @@ set -e # default to build Release -build_type=${1:-Release} +build_type=${1:-Debug} DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" @@ -52,9 +52,9 @@ if [ "$(uname)" == "Darwin" ]; then fi fi elif [ "$(uname)" == "Linux" ]; then - GLIBC_COMPATIBILITY="-DGLIBC_COMPATIBILITY=1" + # GLIBC_COMPATIBILITY="-DGLIBC_COMPATIBILITY=1" UNWIND="-DUSE_UNWIND=1" - JEMALLOC="-DENABLE_JEMALLOC=1" + JEMALLOC="-DENABLE_JEMALLOC=0" PYINIT_ENTRY="-Wl,-ePyInit_${CHDB_PY_MOD}" ICU="-DENABLE_ICU=1" SED_INPLACE="sed -i" @@ -96,6 +96,9 @@ CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${build_type} -DENABLE_THINLTO=0 -DENABLE_TESTS=0 -DENABLE_KAFKA=1 -DENABLE_LIBPQXX=1 -DENABLE_NATS=0 -DENABLE_AMQPCPP=0 -DENABLE_NURAFT=0 \ -DENABLE_CASSANDRA=0 -DENABLE_ODBC=0 -DENABLE_NLP=0 \ -DENABLE_LDAP=0 \ + -DUSE_MUSL=1 \ + -DRust_CARGO_TARGET=x86_64-unknown-linux-musl \ + -DRust_RUSTUP_INSTALL_MISSING_TARGET=ON \ ${MYSQL} \ ${HDFS} \ -DENABLE_LIBRARIES=0 ${RUST_FEATURES} \ @@ -171,71 +174,71 @@ CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${build_type} -DENABLE_THINLTO=0 -DENABLE_TESTS=0 # fi -LIBCHDB_SO="libchdb.so" +# LIBCHDB_SO="libchdb.so" # Build libchdb.so -cmake ${CMAKE_ARGS} -DENABLE_PYTHON=0 .. -ninja -d keeprsp +# cmake ${CMAKE_ARGS} -DENABLE_PYTHON=0 .. +# ninja -d keeprsp BINARY=${BUILD_DIR}/programs/clickhouse -echo -e "\nBINARY: ${BINARY}" -ls -lh ${BINARY} -echo -e "\nldd ${BINARY}" -${LDD} ${BINARY} -rm -f ${BINARY} - -cd ${BUILD_DIR} -ninja -d keeprsp -v > build.log || true -USING_RESPONSE_FILE=$(grep -m 1 'clang++.*-o programs/clickhouse .*' build.log | grep '@CMakeFiles/clickhouse.rsp' || true) - -if [ ! "${USING_RESPONSE_FILE}" == "" ]; then - if [ -f CMakeFiles/clickhouse.rsp ]; then - cp -a CMakeFiles/clickhouse.rsp CMakeFiles/libchdb.rsp - else - echo "CMakeFiles/clickhouse.rsp not found" - exit 1 - fi -fi +# echo -e "\nBINARY: ${BINARY}" +# ls -lh ${BINARY} +# echo -e "\nldd ${BINARY}" +# ${LDD} ${BINARY} || echo "Binary is statically linked (not a dynamic executable)" +# rm -f ${BINARY} + +# cd ${BUILD_DIR} +# ninja -d keeprsp -v > build.log || true +# USING_RESPONSE_FILE=$(grep -m 1 'clang++.*-o programs/clickhouse .*' build.log | grep '@CMakeFiles/clickhouse.rsp' || true) + +# if [ ! "${USING_RESPONSE_FILE}" == "" ]; then +# if [ -f CMakeFiles/clickhouse.rsp ]; then +# cp -a CMakeFiles/clickhouse.rsp CMakeFiles/libchdb.rsp +# else +# echo "CMakeFiles/clickhouse.rsp not found" +# exit 1 +# fi +# fi -LIBCHDB_CMD=$(grep -m 1 'clang++.*-o programs/clickhouse .*' build.log \ - | sed "s/-o programs\/clickhouse/-fPIC -shared -o ${LIBCHDB_SO}/" \ - | sed 's/^[^&]*&& //' | sed 's/&&.*//' \ - | sed 's/ -Wl,-undefined,error/ -Wl,-undefined,dynamic_lookup/g' \ - | sed 's/ -Xlinker --no-undefined//g' \ - | sed 's/@CMakeFiles\/clickhouse.rsp/@CMakeFiles\/libchdb.rsp/g' \ - ) +# LIBCHDB_CMD=$(grep -m 1 'clang++.*-o programs/clickhouse .*' build.log \ +# | sed "s/-o programs\/clickhouse/-fPIC -shared -o ${LIBCHDB_SO}/" \ +# | sed 's/^[^&]*&& //' | sed 's/&&.*//' \ +# | sed 's/ -Wl,-undefined,error/ -Wl,-undefined,dynamic_lookup/g' \ +# | sed 's/ -Xlinker --no-undefined//g' \ +# | sed 's/@CMakeFiles\/clickhouse.rsp/@CMakeFiles\/libchdb.rsp/g' \ +# ) -# generate the command to generate libchdb.so -LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed 's/ '${CHDB_PY_MODULE}'/ '${LIBCHDB_SO}'/g') +# # generate the command to generate libchdb.so +# LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed 's/ '${CHDB_PY_MODULE}'/ '${LIBCHDB_SO}'/g') -if [ ! "${USING_RESPONSE_FILE}" == "" ]; then - ${SED_INPLACE} 's/ '${CHDB_PY_MODULE}'/ '${LIBCHDB_SO}'/g' CMakeFiles/libchdb.rsp -fi +# if [ ! "${USING_RESPONSE_FILE}" == "" ]; then +# ${SED_INPLACE} 's/ '${CHDB_PY_MODULE}'/ '${LIBCHDB_SO}'/g' CMakeFiles/libchdb.rsp +# fi -if [ "$(uname)" == "Linux" ]; then - LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed 's/ '${PYINIT_ENTRY}'/ /g') - if [ ! "${USING_RESPONSE_FILE}" == "" ]; then - ${SED_INPLACE} 's/ '${PYINIT_ENTRY}'/ /g' CMakeFiles/libchdb.rsp - fi -fi +# if [ "$(uname)" == "Linux" ]; then +# LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed 's/ '${PYINIT_ENTRY}'/ /g') +# if [ ! "${USING_RESPONSE_FILE}" == "" ]; then +# ${SED_INPLACE} 's/ '${PYINIT_ENTRY}'/ /g' CMakeFiles/libchdb.rsp +# fi +# fi -if [ "$(uname)" == "Darwin" ]; then - LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed 's/ '${PYINIT_ENTRY}'/ -Wl,-exported_symbol,_query_stable -Wl,-exported_symbol,_free_result -Wl,-exported_symbol,_query_stable_v2 -Wl,-exported_symbol,_free_result_v2/g') - # ${SED_INPLACE} 's/ '${PYINIT_ENTRY}'/ -Wl,-exported_symbol,_query_stable -Wl,-exported_symbol,_free_result -Wl,-exported_symbol,_query_stable_v2 -Wl,-exported_symbol,_free_result_v2/g' CMakeFiles/libchdb.rsp -fi +# if [ "$(uname)" == "Darwin" ]; then +# LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed 's/ '${PYINIT_ENTRY}'/ -Wl,-exported_symbol,_query_stable -Wl,-exported_symbol,_free_result -Wl,-exported_symbol,_query_stable_v2 -Wl,-exported_symbol,_free_result_v2/g') +# # ${SED_INPLACE} 's/ '${PYINIT_ENTRY}'/ -Wl,-exported_symbol,_query_stable -Wl,-exported_symbol,_free_result -Wl,-exported_symbol,_query_stable_v2 -Wl,-exported_symbol,_free_result_v2/g' CMakeFiles/libchdb.rsp +# fi -LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed 's/@CMakeFiles\/clickhouse.rsp/@CMakeFiles\/libchdb.rsp/g') +# LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed 's/@CMakeFiles\/clickhouse.rsp/@CMakeFiles\/libchdb.rsp/g') -# Step 4: -# save the command to a file for debug -echo ${LIBCHDB_CMD} > libchdb_cmd.sh +# # Step 4: +# # save the command to a file for debug +# echo ${LIBCHDB_CMD} > libchdb_cmd.sh -# Step 5: -${LIBCHDB_CMD} +# # Step 5: +# ${LIBCHDB_CMD} LIBCHDB_DIR=${BUILD_DIR}/ -LIBCHDB=${LIBCHDB_DIR}/${LIBCHDB_SO} -ls -lh ${LIBCHDB} +# LIBCHDB=${LIBCHDB_DIR}/${LIBCHDB_SO} +# ls -lh ${LIBCHDB} # build chdb python module py_version="3.8" @@ -309,43 +312,43 @@ ls -lh ${CHDB_PY_MODULE} LIBCHDB_DIR=${BUILD_DIR}/ PYCHDB=${LIBCHDB_DIR}/${CHDB_PY_MODULE} -LIBCHDB=${LIBCHDB_DIR}/${LIBCHDB_SO} +# LIBCHDB=${LIBCHDB_DIR}/${LIBCHDB_SO} if [ ${build_type} == "Debug" ]; then echo -e "\nDebug build, skip strip" else echo -e "\nStrip the binary:" ${STRIP} --remove-section=.comment --remove-section=.note ${PYCHDB} - ${STRIP} --remove-section=.comment --remove-section=.note ${LIBCHDB} + # ${STRIP} --remove-section=.comment --remove-section=.note ${LIBCHDB} fi echo -e "\nStripe the binary:" echo -e "\nPYCHDB: ${PYCHDB}" ls -lh ${PYCHDB} -echo -e "\nLIBCHDB: ${LIBCHDB}" -ls -lh ${LIBCHDB} +# echo -e "\nLIBCHDB: ${LIBCHDB}" +# ls -lh ${LIBCHDB} echo -e "\nldd ${PYCHDB}" -${LDD} ${PYCHDB} +${LDD} ${PYCHDB} || echo "Binary is statically linked (not a dynamic executable)" echo -e "\nfile info of ${PYCHDB}" file ${PYCHDB} -echo -e "\nldd ${LIBCHDB}" -${LDD} ${LIBCHDB} -echo -e "\nfile info of ${LIBCHDB}" -file ${LIBCHDB} +# echo -e "\nldd ${LIBCHDB}" +# ${LDD} ${LIBCHDB} || echo "Binary is statically linked (not a dynamic executable)" +# echo -e "\nfile info of ${LIBCHDB}" +# file ${LIBCHDB} rm -f ${CHDB_DIR}/*.so cp -a ${PYCHDB} ${CHDB_DIR}/${CHDB_PY_MODULE} -cp -a ${LIBCHDB} ${PROJ_DIR}/${LIBCHDB_SO} +# cp -a ${LIBCHDB} ${PROJ_DIR}/${LIBCHDB_SO} echo -e "\nSymbols:" echo -e "\nPyInit in PYCHDB: ${PYCHDB}" ${NM} ${PYCHDB} | grep PyInit || true -echo -e "\nPyInit in LIBCHDB: ${LIBCHDB}" -${NM} ${LIBCHDB} | grep PyInit || echo "PyInit not found in ${LIBCHDB}, it's OK" +# echo -e "\nPyInit in LIBCHDB: ${LIBCHDB}" +# ${NM} ${LIBCHDB} | grep PyInit || echo "PyInit not found in ${LIBCHDB}, it's OK" echo -e "\nquery_stable in PYCHDB: ${PYCHDB}" ${NM} ${PYCHDB} | grep query_stable || true -echo -e "\nquery_stable in LIBCHDB: ${LIBCHDB}" -${NM} ${LIBCHDB} | grep query_stable || true +# echo -e "\nquery_stable in LIBCHDB: ${LIBCHDB}" +# ${NM} ${LIBCHDB} | grep query_stable || true echo -e "\nAfter copy:" cd ${PROJ_DIR} && pwd diff --git a/chdb/vars.sh b/chdb/vars.sh index b1b2100a5b6..19392da05bf 100755 --- a/chdb/vars.sh +++ b/chdb/vars.sh @@ -6,7 +6,8 @@ CHDB_DIR="$PROJ_DIR/chdb" # chdb directory CHDB_PY_MOD="_chdb" CHDB_PY_MODULE="${CHDB_PY_MOD}.abi3.so" pushd ${PROJ_DIR} -CHDB_VERSION=$(python3 -c 'import setup; print(setup.get_latest_git_tag())') +# CHDB_VERSION=$(python3 -c 'import setup; print(setup.get_latest_git_tag())') +CHDB_VERSION='3.6.0' popd # try to use largest llvm-strip version diff --git a/cmake/cpu_features.cmake b/cmake/cpu_features.cmake index fa58a739611..ddf9d940e15 100644 --- a/cmake/cpu_features.cmake +++ b/cmake/cpu_features.cmake @@ -156,20 +156,20 @@ elseif (ARCH_AMD64) endif() # Same best-effort check for x86 as above for ARM. - if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "amd64|x86_64" AND NOT NO_SSE3_OR_HIGHER) - # Test for flags in standard profile but not in NO_SSE3_OR_HIGHER profile. - # /proc/cpuid for Intel Xeon 8124: "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse - # sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf - # tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c - # rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx - # avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke"" - execute_process( - COMMAND grep -P "^(?=.*ssse3)(?=.*sse4_1)(?=.*sse4_2)" /proc/cpuinfo - OUTPUT_VARIABLE FLAGS) - if (NOT FLAGS) - MESSAGE(FATAL_ERROR "The build machine does not satisfy the minimum CPU requirements, try to run cmake with -DNO_SSE3_OR_HIGHER=1") - endif() - endif() + # if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "amd64|x86_64" AND NOT NO_SSE3_OR_HIGHER) + # # Test for flags in standard profile but not in NO_SSE3_OR_HIGHER profile. + # # /proc/cpuid for Intel Xeon 8124: "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse + # # sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf + # # tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c + # # rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx + # # avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke"" + # execute_process( + # COMMAND grep -P "^(?=.*ssse3)(?=.*sse4_1)(?=.*sse4_2)" /proc/cpuinfo + # OUTPUT_VARIABLE FLAGS) + # if (NOT FLAGS) + # MESSAGE(FATAL_ERROR "The build machine does not satisfy the minimum CPU requirements, try to run cmake with -DNO_SSE3_OR_HIGHER=1") + # endif() + # endif() # ClickHouse can be cross-compiled (e.g. on an ARM host for x86) but it is also possible to build ClickHouse on x86 w/o AVX for x86 w/ # AVX. We only assume that the compiler can emit certain SIMD instructions, we don't care if the host system is able to run the binary. diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h index c20ce45b684..3a3615c948f 100644 --- a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h +++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h @@ -24,14 +24,14 @@ * Define overrides for non-standard allocator-related functions if they are * present on the system. */ -#if !defined(USE_MUSL) +// #if !defined(USE_MUSL) #define JEMALLOC_OVERRIDE_MEMALIGN #define JEMALLOC_OVERRIDE_VALLOC #if defined(__linux__) #define JEMALLOC_OVERRIDE_PVALLOC #define JEMALLOC_OVERRIDE___LIBC_PVALLOC #endif -#endif +// #endif /* * At least Linux omits the "const" in: diff --git a/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in index 729bd82c2fa..dfba1c282aa 100644 --- a/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in @@ -140,7 +140,8 @@ /* #undef JEMALLOC_MUTEX_INIT_CB */ /* Non-empty if the tls_model attribute is supported. */ -#define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) +// #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) +#define JEMALLOC_TLS_MODEL __attribute__((tls_model("global-dynamic"))) /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables @@ -375,10 +376,10 @@ #define LG_SIZEOF_INTMAX_T 3 /* glibc malloc hooks (__malloc_hook, __realloc_hook, __free_hook). */ -#define JEMALLOC_GLIBC_MALLOC_HOOK +// #define JEMALLOC_GLIBC_MALLOC_HOOK /* glibc memalign hook. */ -#define JEMALLOC_GLIBC_MEMALIGN_HOOK +// #define JEMALLOC_GLIBC_MEMALIGN_HOOK /* pthread support */ #define JEMALLOC_HAVE_PTHREAD diff --git a/contrib/pybind11 b/contrib/pybind11 index e4b03842254..b1c2263dc37 160000 --- a/contrib/pybind11 +++ b/contrib/pybind11 @@ -1 +1 @@ -Subproject commit e4b03842254775bbb7b8567376787e6eb04ebfd3 +Subproject commit b1c2263dc37c7b3aec9c210bf1beb8b83dbbb995 diff --git a/contrib/pybind11-cmake/CMakeLists.txt b/contrib/pybind11-cmake/CMakeLists.txt index 42d577b8205..d87aef848ab 100644 --- a/contrib/pybind11-cmake/CMakeLists.txt +++ b/contrib/pybind11-cmake/CMakeLists.txt @@ -7,7 +7,48 @@ endif() string(REPLACE "-Wl,-z,defs" "" CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS}") if (DEFINED PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION) - find_package(Python ${PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION} EXACT REQUIRED COMPONENTS Interpreter Development) + # Set Python root hint for pyenv installations + if(DEFINED ENV{PYENV_VERSION}) + set(Python_ROOT_DIR "$ENV{PYENV_ROOT}/versions/$ENV{PYENV_VERSION}") + message(STATUS "Using Python_ROOT_DIR: ${Python_ROOT_DIR}") + endif() + + # First try to find with EXACT version + find_package(Python ${PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION} EXACT COMPONENTS Interpreter Development) + + if(NOT Python_FOUND) + message(STATUS "Exact Python ${PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION} not found, trying with manual hints...") + + # Try to manually set paths if pyenv is being used + if(DEFINED ENV{PYENV_VERSION}) + set(Python_EXECUTABLE "$ENV{PYENV_ROOT}/versions/$ENV{PYENV_VERSION}/bin/python") + set(Python_INCLUDE_DIRS "$ENV{PYENV_ROOT}/versions/$ENV{PYENV_VERSION}/include/python${PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION}") + set(Python_LIBRARY_DIRS "$ENV{PYENV_ROOT}/versions/$ENV{PYENV_VERSION}/lib") + + # Try to find the library file + find_library(Python_LIBRARIES + NAMES python${PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION} python${PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION}m + PATHS "${Python_LIBRARY_DIRS}" + NO_DEFAULT_PATH + ) + + if(Python_EXECUTABLE AND Python_INCLUDE_DIRS AND Python_LIBRARIES) + set(Python_FOUND TRUE) + set(Python_VERSION ${PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION}) + string(REPLACE "." ";" VERSION_LIST ${Python_VERSION}) + list(GET VERSION_LIST 0 Python_VERSION_MAJOR) + list(GET VERSION_LIST 1 Python_VERSION_MINOR) + message(STATUS "Manually configured Python ${Python_VERSION}") + message(STATUS " Executable: ${Python_EXECUTABLE}") + message(STATUS " Include dirs: ${Python_INCLUDE_DIRS}") + message(STATUS " Libraries: ${Python_LIBRARIES}") + endif() + endif() + + if(NOT Python_FOUND) + message(FATAL_ERROR "Could NOT find Python ${PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION} development files") + endif() + endif() else() find_package(Python REQUIRED COMPONENTS Interpreter Development) endif() @@ -23,7 +64,11 @@ set(STUBS_SOURCES "${LIBRARY_DIR}/source/non_limited_api/non_limited_api_stubs.cpp" ) -add_library(pybind11_stubs SHARED ${STUBS_SOURCES}) +if (NOT USE_MUSL) + add_library(pybind11_stubs SHARED ${STUBS_SOURCES}) +else() + add_library(pybind11_stubs STATIC ${STUBS_SOURCES}) +endif() target_include_directories(pybind11_stubs PRIVATE ${PYBIND11_INCLUDE_DIR}) target_include_directories(pybind11_stubs PRIVATE ${Python_INCLUDE_DIRS}) target_compile_definitions(pybind11_stubs PUBLIC @@ -43,6 +88,10 @@ if (APPLE) target_link_options(pybind11_stubs PRIVATE -undefined dynamic_lookup) else() target_link_options(pybind11_stubs PRIVATE -Wl,--unresolved-symbols=ignore-all) + # For musl cross-compilation, add nostartfiles to avoid crtbeginS.o/crtendS.o issues + if (USE_MUSL) + target_link_options(pybind11_stubs PRIVATE -nostartfiles) + endif() endif() add_library(ch_contrib::pybind11_stubs ALIAS pybind11_stubs) @@ -73,6 +122,10 @@ if(Python_FOUND) target_link_options(${PYBIND11_NONLIMITEDAPI_LIBNAME} PRIVATE -undefined dynamic_lookup) else() target_link_options(${PYBIND11_NONLIMITEDAPI_LIBNAME} PRIVATE -Wl,--unresolved-symbols=ignore-all) + # For musl cross-compilation, add nostartfiles to avoid crtbeginS.o/crtendS.o issues + if (USE_MUSL) + target_link_options(${PYBIND11_NONLIMITEDAPI_LIBNAME} PRIVATE -nostartfiles) + endif() endif() else() message(FATAL_ERROR "Python ${PY_VER} not found, aborting build") diff --git a/setup.py b/setup.py index 9b3d601f389..84b2836a6b7 100644 --- a/setup.py +++ b/setup.py @@ -145,7 +145,8 @@ def build_extensions(self): ), ] # fix the version in chdb/__init__.py - versionStr = get_latest_git_tag() + # versionStr = get_latest_git_tag() + versionStr = "3.6.0" # Call the function to update pyproject.toml # update_pyproject_version(versionStr) diff --git a/src/Access/Common/AccessFlags.cpp b/src/Access/Common/AccessFlags.cpp index 9d35bac958e..3d3fe2d3732 100644 --- a/src/Access/Common/AccessFlags.cpp +++ b/src/Access/Common/AccessFlags.cpp @@ -266,8 +266,15 @@ namespace start_node->aliases.emplace_back(start_node->keyword); for (auto & alias : start_node->aliases) { - boost::to_upper(alias); - keyword_to_flags_map[alias] = start_node->flags; + // boost::to_upper(alias); + // keyword_to_flags_map[alias] = start_node->flags; + std::string upper_alias = alias; + for (char & c : upper_alias) + { + if (c >= 'a' && c <= 'z') + c = c - 'a' + 'A'; + } + keyword_to_flags_map[upper_alias] = start_node->flags; } for (auto & child : start_node->children) diff --git a/src/Common/AllocationInterceptors.cpp b/src/Common/AllocationInterceptors.cpp index 52d71efe319..e49f784797d 100644 --- a/src/Common/AllocationInterceptors.cpp +++ b/src/Common/AllocationInterceptors.cpp @@ -44,6 +44,65 @@ static struct InitializeJemallocZoneAllocatorForOSX } initializeJemallocZoneAllocatorForOSX; #endif +#if USE_JEMALLOC + +namespace Memory +{ + thread_local bool disable_memory_check = false; +} + +#endif + +ALWAYS_INLINE void * tryNewNonJemallocMemoryConditional(std::size_t size) +{ +#if USE_JEMALLOC + if (likely(!Memory::disable_memory_check)) + { + return Memory::realNewImpl(size); + } +#endif + (void)size; + return nullptr; +} + +ALWAYS_INLINE void * tryNewNonJemallocMemoryConditional(std::size_t size, std::align_val_t align) +{ +#if USE_JEMALLOC + if (likely(!Memory::disable_memory_check)) + { + return Memory::realNewImpl(size, align); + } +#endif + (void)size; + (void)align; + return nullptr; +} + +ALWAYS_INLINE void * tryNewNoExceptNonJemallocMemoryConditional(std::size_t size) +{ +#if USE_JEMALLOC + if (likely(!Memory::disable_memory_check)) + { + return Memory::realNewNoExcept(size); + } +#endif + (void)size; + return nullptr; +} + +ALWAYS_INLINE void * tryNewNoExceptNonJemallocMemoryConditional(std::size_t size, std::align_val_t align) +{ +#if USE_JEMALLOC + if (likely(!Memory::disable_memory_check)) + { + return Memory::realNewNoExcept(size, align); + } +#endif + (void)size; + (void)align; + return nullptr; +} + /// Replace default new/delete with memory tracking versions. /// @sa https://en.cppreference.com/w/cpp/memory/new/operator_new /// https://en.cppreference.com/w/cpp/memory/new/operator_delete @@ -52,6 +111,9 @@ static struct InitializeJemallocZoneAllocatorForOSX void * operator new(std::size_t size) { + if (void * ptr = tryNewNonJemallocMemoryConditional(size)) + return ptr; + AllocationTrace trace; std::size_t actual_size = Memory::trackMemory(size, trace); void * ptr = Memory::newImpl(size); @@ -61,6 +123,9 @@ void * operator new(std::size_t size) void * operator new(std::size_t size, std::align_val_t align) { + if (void * ptr = tryNewNonJemallocMemoryConditional(size, align)) + return ptr; + AllocationTrace trace; std::size_t actual_size = Memory::trackMemory(size, trace, align); void * ptr = Memory::newImpl(size, align); @@ -70,6 +135,9 @@ void * operator new(std::size_t size, std::align_val_t align) void * operator new[](std::size_t size) { + if (void * ptr = tryNewNonJemallocMemoryConditional(size)) + return ptr; + AllocationTrace trace; std::size_t actual_size = Memory::trackMemory(size, trace); void * ptr = Memory::newImpl(size); @@ -79,6 +147,9 @@ void * operator new[](std::size_t size) void * operator new[](std::size_t size, std::align_val_t align) { + if (void * ptr = tryNewNonJemallocMemoryConditional(size, align)) + return ptr; + AllocationTrace trace; std::size_t actual_size = Memory::trackMemory(size, trace, align); void * ptr = Memory::newImpl(size, align); @@ -88,6 +159,9 @@ void * operator new[](std::size_t size, std::align_val_t align) void * operator new(std::size_t size, const std::nothrow_t &) noexcept { + if (void * ptr = tryNewNoExceptNonJemallocMemoryConditional(size)) + return ptr; + AllocationTrace trace; std::size_t actual_size = Memory::trackMemory(size, trace); void * ptr = Memory::newNoExcept(size); @@ -97,6 +171,9 @@ void * operator new(std::size_t size, const std::nothrow_t &) noexcept void * operator new[](std::size_t size, const std::nothrow_t &) noexcept { + if (void * ptr = tryNewNoExceptNonJemallocMemoryConditional(size)) + return ptr; + AllocationTrace trace; std::size_t actual_size = Memory::trackMemory(size, trace); void * ptr = Memory::newNoExcept(size); @@ -106,6 +183,9 @@ void * operator new[](std::size_t size, const std::nothrow_t &) noexcept void * operator new(std::size_t size, std::align_val_t align, const std::nothrow_t &) noexcept { + if (void * ptr = tryNewNoExceptNonJemallocMemoryConditional(size, align)) + return ptr; + AllocationTrace trace; std::size_t actual_size = Memory::trackMemory(size, trace, align); void * ptr = Memory::newNoExcept(size, align); @@ -115,6 +195,9 @@ void * operator new(std::size_t size, std::align_val_t align, const std::nothrow void * operator new[](std::size_t size, std::align_val_t align, const std::nothrow_t &) noexcept { + if (void * ptr = tryNewNoExceptNonJemallocMemoryConditional(size, align)) + return ptr; + AllocationTrace trace; std::size_t actual_size = Memory::trackMemory(size, trace, align); void * ptr = Memory::newNoExcept(size, align); @@ -126,7 +209,7 @@ void * operator new[](std::size_t size, std::align_val_t align, const std::nothr extern "C" void __real_free(void * ptr); -inline ALWAYS_INLINE bool isJemallocMemory(void * ptr) +ALWAYS_INLINE bool isJemallocMemory(void * ptr) { int arena_ind = je_mallctl("arenas.lookup", nullptr, nullptr, &ptr, sizeof(ptr)); return arena_ind == 0; // arena_ind == 0 means jemalloc memory @@ -145,7 +228,7 @@ inline ALWAYS_INLINE bool isJemallocMemory(void * ptr) /// /// Note: We don't update memory tracking for non-jemalloc memory since it was likely /// never tracked by our system in the first place. -inline ALWAYS_INLINE bool tryFreeNonJemallocMemory(void * ptr) +ALWAYS_INLINE bool tryFreeNonJemallocMemory(void * ptr) { if (unlikely(ptr == nullptr)) return true; @@ -160,12 +243,7 @@ inline ALWAYS_INLINE bool tryFreeNonJemallocMemory(void * ptr) return false; // Not handled - should continue with jemalloc path } -namespace Memory -{ -thread_local bool disable_memory_check{false}; -} - -inline ALWAYS_INLINE bool tryFreeNonJemallocMemoryConditional(void * ptr) +ALWAYS_INLINE bool tryFreeNonJemallocMemoryConditional(void * ptr) { if (unlikely(ptr == nullptr)) return true; diff --git a/src/Common/memory.h b/src/Common/memory.h index cbe9f7b7c64..143a4e34182 100644 --- a/src/Common/memory.h +++ b/src/Common/memory.h @@ -87,6 +87,23 @@ inline ALWAYS_INLINE void * newImpl(std::size_t size, TAlign... align) throw std::bad_alloc{}; } +template ... TAlign> +requires DB::OptionalArgument +inline ALWAYS_INLINE void * realNewImpl(std::size_t size, TAlign... align) +{ + void * ptr = nullptr; + if constexpr (sizeof...(TAlign) == 1) + ptr = __real_aligned_alloc(alignToSizeT(align...), alignUp(size, alignToSizeT(align...))); + else + ptr = __real_malloc(size); + + if (likely(ptr != nullptr)) + return ptr; + + /// @note no std::get_new_handler logic implemented + throw std::bad_alloc{}; +} + #else template ... TAlign> requires DB::OptionalArgument @@ -122,6 +139,16 @@ inline ALWAYS_INLINE void deleteImpl(void * ptr) noexcept je_free(ptr); } +inline ALWAYS_INLINE void * realNewNoExcept(std::size_t size) noexcept +{ + return __real_malloc(size); +} + +inline ALWAYS_INLINE void * realNewNoExcept(std::size_t size, std::align_val_t align) noexcept +{ + return __real_aligned_alloc(static_cast(align), size); +} + #else inline ALWAYS_INLINE void * newNoExcept(std::size_t size) noexcept From 4064fda8803ebb893f72153f65e8de552014f800 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Mon, 13 Oct 2025 21:25:47 +0800 Subject: [PATCH 02/41] chore: update build.sh --- CMakeLists.txt | 1 - PreLoad.cmake | 4 +- chdb/build-musl.sh | 166 ++++++++++++++++++ cmake/cpu_features.cmake | 30 ++-- .../include/jemalloc/jemalloc_defs.h | 4 +- .../internal/jemalloc_internal_defs.h.in | 7 +- contrib/pybind11-cmake/CMakeLists.txt | 45 +---- src/Access/Common/AccessFlags.cpp | 11 +- src/Common/AllocationInterceptors.cpp | 94 +--------- src/Common/memory.h | 27 --- 10 files changed, 200 insertions(+), 189 deletions(-) create mode 100755 chdb/build-musl.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 37c8bc799ee..f798d19698c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -163,7 +163,6 @@ if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWIT set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -fuse-ctor-homing") endif() -# Fix TLS model for MUSL to avoid initial-exec TLS relocation errors if (USE_MUSL) add_definitions(-DUSE_MUSL=1 -D__MUSL__=1) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ftls-model=global-dynamic") diff --git a/PreLoad.cmake b/PreLoad.cmake index 0ac0a8b14be..0e13f3b541c 100644 --- a/PreLoad.cmake +++ b/PreLoad.cmake @@ -91,7 +91,9 @@ if (OS MATCHES "Linux" # - compile musl with debug and -fasynchronous-unwind-tables # # But none of this changes anything so far. - # set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-x86_64.cmake" CACHE INTERNAL "") + if ("$ENV{USE_MUSL}" STREQUAL "") + set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-x86_64.cmake" CACHE INTERNAL "") + endif() elseif (ARCH MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)") set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-aarch64.cmake" CACHE INTERNAL "") elseif (ARCH MATCHES "^(ppc64le.*|PPC64LE.*)") diff --git a/chdb/build-musl.sh b/chdb/build-musl.sh new file mode 100755 index 00000000000..208a491e692 --- /dev/null +++ b/chdb/build-musl.sh @@ -0,0 +1,166 @@ +#!/bin/bash + +set -e + +export USE_MUSL=1 + +build_type=${1:-Release} + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +. ${DIR}/vars.sh + +BUILD_DIR=${PROJ_DIR}/buildlib + +HDFS="-DENABLE_HDFS=1 -DENABLE_GSASL_LIBRARY=1 -DENABLE_KRB5=1" +MYSQL="-DENABLE_MYSQL=1" +RUST_FEATURES="-DENABLE_RUST=0" +if [ "$(uname)" == "Linux" ]; then + GLIBC_COMPATIBILITY="-DGLIBC_COMPATIBILITY=0" + UNWIND="-DUSE_UNWIND=1" + JEMALLOC="-DENABLE_JEMALLOC=0" + PYINIT_ENTRY="-Wl,-ePyInit_${CHDB_PY_MOD}" + ICU="-DENABLE_ICU=1" + SED_INPLACE="sed -i" + # only x86_64, enable AVX, enable embedded compiler + if [ "$(uname -m)" == "x86_64" ]; then + CPU_FEATURES="-DENABLE_AVX=1 -DENABLE_AVX2=0" + LLVM="-DENABLE_EMBEDDED_COMPILER=1 -DENABLE_DWARF_PARSER=1" + RUST_FEATURES="-DENABLE_RUST=1 -DENABLE_DELTA_KERNEL_RS=1" + CORROSION_CMAKE_FILE="${PROJ_DIR}/contrib/corrosion-cmake/CMakeLists.txt" + if [ -f "${CORROSION_CMAKE_FILE}" ]; then + if ! grep -q 'OPENSSL_NO_DEPRECATED_3_0' "${CORROSION_CMAKE_FILE}"; then + echo "Modifying corrosion CMakeLists.txt for Linux x86_64..." + ${SED_INPLACE} 's/corrosion_set_env_vars(${target_name} "RUSTFLAGS=${RUSTFLAGS}")/corrosion_set_env_vars(${target_name} "RUSTFLAGS=${RUSTFLAGS} --cfg osslconf=\\\"OPENSSL_NO_DEPRECATED_3_0\\\"")/g' "${CORROSION_CMAKE_FILE}" + else + echo "corrosion CMakeLists.txt already modified, skipping..." + fi + else + echo "Warning: corrosion CMakeLists.txt not found at ${CORROSION_CMAKE_FILE}" + fi + else + CPU_FEATURES="-DENABLE_AVX=0 -DENABLE_AVX2=0 -DNO_ARMV81_OR_HIGHER=1" + LLVM="-DENABLE_EMBEDDED_COMPILER=0 -DENABLE_DWARF_PARSER=0" + fi +else + echo "OS not supported" + exit 1 +fi + +if [ ! -d $BUILD_DIR ]; then + mkdir $BUILD_DIR +fi + +cd ${BUILD_DIR} +CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${build_type} -DENABLE_THINLTO=0 -DENABLE_TESTS=0 -DENABLE_CLICKHOUSE_SERVER=0 -DENABLE_CLICKHOUSE_CLIENT=0 \ + -DENABLE_CLICKHOUSE_KEEPER=0 -DENABLE_CLICKHOUSE_KEEPER_CONVERTER=0 -DENABLE_CLICKHOUSE_LOCAL=1 -DENABLE_CLICKHOUSE_SU=0 -DENABLE_CLICKHOUSE_BENCHMARK=0 \ + -DENABLE_AZURE_BLOB_STORAGE=1 -DENABLE_CLICKHOUSE_COPIER=0 -DENABLE_CLICKHOUSE_DISKS=0 -DENABLE_CLICKHOUSE_FORMAT=0 -DENABLE_CLICKHOUSE_GIT_IMPORT=0 \ + -DENABLE_AWS_S3=1 -DENABLE_HIVE=0 -DENABLE_AVRO=1 \ + -DENABLE_CLICKHOUSE_OBFUSCATOR=0 -DENABLE_CLICKHOUSE_ODBC_BRIDGE=0 -DENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER=0 \ + -DENABLE_KAFKA=1 -DENABLE_LIBPQXX=1 -DENABLE_NATS=0 -DENABLE_AMQPCPP=0 -DENABLE_NURAFT=0 \ + -DENABLE_CASSANDRA=0 -DENABLE_ODBC=0 -DENABLE_NLP=0 \ + -DENABLE_LDAP=0 \ + -DUSE_MUSL=1 \ + -DRust_RUSTUP_INSTALL_MISSING_TARGET=ON \ + ${MYSQL} \ + ${HDFS} \ + -DENABLE_LIBRARIES=0 ${RUST_FEATURES} \ + ${GLIBC_COMPATIBILITY} \ + -DENABLE_UTILS=0 ${LLVM} ${UNWIND} \ + ${ICU} -DENABLE_UTF8PROC=1 ${JEMALLOC} \ + -DENABLE_PARQUET=1 -DENABLE_ROCKSDB=1 -DENABLE_SQLITE=1 -DENABLE_VECTORSCAN=1 \ + -DENABLE_PROTOBUF=1 -DENABLE_THRIFT=1 -DENABLE_MSGPACK=1 \ + -DENABLE_BROTLI=1 -DENABLE_H3=1 -DENABLE_CURL=1 \ + -DENABLE_CLICKHOUSE_ALL=0 -DUSE_STATIC_LIBRARIES=1 -DSPLIT_SHARED_LIBRARIES=0 \ + -DENABLE_SIMDJSON=1 -DENABLE_RAPIDJSON=1 \ + ${CPU_FEATURES} \ + -DENABLE_AVX512=0 -DENABLE_AVX512_VBMI=0 \ + -DENABLE_LIBFIU=1 \ + ${COMPILER_CACHE} \ + -DCHDB_VERSION=${CHDB_VERSION} \ + " + +BINARY=${BUILD_DIR}/programs/clickhouse + +# build chdb python module +py_version="3.8" +current_py_version=$(python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')") +if [ "$current_py_version" != "$py_version" ]; then + echo "Error: Current Python version is $current_py_version, but required version is $py_version" + echo "Please switch to Python $py_version using: pyenv shell $py_version" + exit 1 +fi +echo "Using Python version: $current_py_version" +cmake ${CMAKE_ARGS} -DENABLE_PYTHON=1 -DPYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION=${py_version} .. +ninja -d keeprsp || true + +# del the binary and run ninja -v again to capture the command, then modify it to generate CHDB_PY_MODULE +/bin/rm -f ${BINARY} +cd ${BUILD_DIR} +ninja -d keeprsp -v > build.log || true + +USING_RESPONSE_FILE=$(grep -m 1 'clang++.*-o programs/clickhouse .*' build.log | grep '@CMakeFiles/clickhouse.rsp' || true) + +if [ ! "${USING_RESPONSE_FILE}" == "" ]; then + if [ -f CMakeFiles/clickhouse.rsp ]; then + cp -a CMakeFiles/clickhouse.rsp CMakeFiles/pychdb.rsp + else + echo "CMakeFiles/clickhouse.rsp not found" + exit 1 + fi +fi + +# extract the command to generate CHDB_PY_MODULE +PYCHDB_CMD=$(grep -m 1 'clang++.*-o programs/clickhouse .*' build.log \ + | sed "s/-o programs\/clickhouse/-fPIC -Wl,-undefined,dynamic_lookup -shared ${PYINIT_ENTRY} -o ${CHDB_PY_MODULE}/" \ + | sed 's/^[^&]*&& //' | sed 's/&&.*//' \ + | sed 's/ -Wl,-undefined,error/ -Wl,-undefined,dynamic_lookup/g' \ + | sed 's/ -Xlinker --no-undefined//g' \ + | sed 's/@CMakeFiles\/clickhouse.rsp/@CMakeFiles\/pychdb.rsp/g' \ + ) + +PYCHDB_CMD=$(echo ${PYCHDB_CMD} | sed 's/ src\/CMakeFiles\/clickhouse_malloc.dir\/Common\/stubFree.c.o//g') +if [ ! "${USING_RESPONSE_FILE}" == "" ]; then + ${SED_INPLACE} 's/ src\/CMakeFiles\/clickhouse_malloc.dir\/Common\/stubFree.c.o//g' CMakeFiles/pychdb.rsp +fi + +PYCHDB_CMD=$(echo ${PYCHDB_CMD} | sed 's|-Wl,-rpath,/[^[:space:]]*/pybind11-cmake|-Wl,-rpath,\$ORIGIN|g') + +echo ${PYCHDB_CMD} > pychdb_cmd.sh + +${PYCHDB_CMD} + +ls -lh ${CHDB_PY_MODULE} + +PYCHDB=${BUILD_DIR}/${CHDB_PY_MODULE} + +if [ ${build_type} == "Debug" ]; then + echo -e "\nDebug build, skip strip" +else + echo -e "\nStrip the binary:" + ${STRIP} --remove-section=.comment --remove-section=.note ${PYCHDB} +fi +echo -e "\nStripe the binary:" + +echo -e "\nPYCHDB: ${PYCHDB}" +ls -lh ${PYCHDB} +echo -e "\nldd ${PYCHDB}" +${LDD} ${PYCHDB} || echo "Binary is statically linked (not a dynamic executable)" +echo -e "\nfile info of ${PYCHDB}" +file ${PYCHDB} + +rm -f ${CHDB_DIR}/*.so +cp -a ${PYCHDB} ${CHDB_DIR}/${CHDB_PY_MODULE} + +echo -e "\nSymbols:" +echo -e "\nPyInit in PYCHDB: ${PYCHDB}" +${NM} ${PYCHDB} | grep PyInit || true +echo -e "\nquery_stable in PYCHDB: ${PYCHDB}" +${NM} ${PYCHDB} | grep query_stable || true + +echo -e "\nAfter copy:" +cd ${PROJ_DIR} && pwd + +ccache -s || true + +CMAKE_ARGS="${CMAKE_ARGS}" bash ${DIR}/build_pybind11.sh --all diff --git a/cmake/cpu_features.cmake b/cmake/cpu_features.cmake index ddf9d940e15..264157aa86c 100644 --- a/cmake/cpu_features.cmake +++ b/cmake/cpu_features.cmake @@ -98,7 +98,7 @@ elseif (ARCH_AARCH64) # and the build machine is too old, i.e. doesn't satisfy above modern profile, then these intermediate binaries will not run (dump # SIGILL). Even if they could run, the build machine wouldn't be able to run the ClickHouse binary. In that case, suggest to run the # build with the compat profile. - if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)" AND NOT NO_ARMV81_OR_HIGHER) + if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)" AND NOT NO_ARMV81_OR_HIGHER AND NOT USE_MUSL) # CPU features in /proc/cpuinfo and compiler flags don't align :( ... pick some obvious flags contained in the modern but not in the # legacy profile (full Graviton 3 /proc/cpuinfo is "fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm # jscvt fcma lrcpc dcpop sha3 sm3 sm4 asimddp sha512 sve asimdfhm dit uscat ilrcpc flagm ssbs paca pacg dcpodp svei8mm svebf16 i8mm @@ -156,20 +156,20 @@ elseif (ARCH_AMD64) endif() # Same best-effort check for x86 as above for ARM. - # if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "amd64|x86_64" AND NOT NO_SSE3_OR_HIGHER) - # # Test for flags in standard profile but not in NO_SSE3_OR_HIGHER profile. - # # /proc/cpuid for Intel Xeon 8124: "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse - # # sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf - # # tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c - # # rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx - # # avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke"" - # execute_process( - # COMMAND grep -P "^(?=.*ssse3)(?=.*sse4_1)(?=.*sse4_2)" /proc/cpuinfo - # OUTPUT_VARIABLE FLAGS) - # if (NOT FLAGS) - # MESSAGE(FATAL_ERROR "The build machine does not satisfy the minimum CPU requirements, try to run cmake with -DNO_SSE3_OR_HIGHER=1") - # endif() - # endif() + if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "amd64|x86_64" AND NOT NO_SSE3_OR_HIGHER AND NOT USE_MUSL) + # Test for flags in standard profile but not in NO_SSE3_OR_HIGHER profile. + # /proc/cpuid for Intel Xeon 8124: "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse + # sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf + # tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c + # rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx + # avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke"" + execute_process( + COMMAND grep -P "^(?=.*ssse3)(?=.*sse4_1)(?=.*sse4_2)" /proc/cpuinfo + OUTPUT_VARIABLE FLAGS) + if (NOT FLAGS) + MESSAGE(FATAL_ERROR "The build machine does not satisfy the minimum CPU requirements, try to run cmake with -DNO_SSE3_OR_HIGHER=1") + endif() + endif() # ClickHouse can be cross-compiled (e.g. on an ARM host for x86) but it is also possible to build ClickHouse on x86 w/o AVX for x86 w/ # AVX. We only assume that the compiler can emit certain SIMD instructions, we don't care if the host system is able to run the binary. diff --git a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h index 3a3615c948f..c20ce45b684 100644 --- a/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h +++ b/contrib/jemalloc-cmake/include/jemalloc/jemalloc_defs.h @@ -24,14 +24,14 @@ * Define overrides for non-standard allocator-related functions if they are * present on the system. */ -// #if !defined(USE_MUSL) +#if !defined(USE_MUSL) #define JEMALLOC_OVERRIDE_MEMALIGN #define JEMALLOC_OVERRIDE_VALLOC #if defined(__linux__) #define JEMALLOC_OVERRIDE_PVALLOC #define JEMALLOC_OVERRIDE___LIBC_PVALLOC #endif -// #endif +#endif /* * At least Linux omits the "const" in: diff --git a/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in index dfba1c282aa..729bd82c2fa 100644 --- a/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in @@ -140,8 +140,7 @@ /* #undef JEMALLOC_MUTEX_INIT_CB */ /* Non-empty if the tls_model attribute is supported. */ -// #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) -#define JEMALLOC_TLS_MODEL __attribute__((tls_model("global-dynamic"))) +#define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables @@ -376,10 +375,10 @@ #define LG_SIZEOF_INTMAX_T 3 /* glibc malloc hooks (__malloc_hook, __realloc_hook, __free_hook). */ -// #define JEMALLOC_GLIBC_MALLOC_HOOK +#define JEMALLOC_GLIBC_MALLOC_HOOK /* glibc memalign hook. */ -// #define JEMALLOC_GLIBC_MEMALIGN_HOOK +#define JEMALLOC_GLIBC_MEMALIGN_HOOK /* pthread support */ #define JEMALLOC_HAVE_PTHREAD diff --git a/contrib/pybind11-cmake/CMakeLists.txt b/contrib/pybind11-cmake/CMakeLists.txt index d87aef848ab..c986799fd10 100644 --- a/contrib/pybind11-cmake/CMakeLists.txt +++ b/contrib/pybind11-cmake/CMakeLists.txt @@ -7,48 +7,7 @@ endif() string(REPLACE "-Wl,-z,defs" "" CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS}") if (DEFINED PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION) - # Set Python root hint for pyenv installations - if(DEFINED ENV{PYENV_VERSION}) - set(Python_ROOT_DIR "$ENV{PYENV_ROOT}/versions/$ENV{PYENV_VERSION}") - message(STATUS "Using Python_ROOT_DIR: ${Python_ROOT_DIR}") - endif() - - # First try to find with EXACT version - find_package(Python ${PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION} EXACT COMPONENTS Interpreter Development) - - if(NOT Python_FOUND) - message(STATUS "Exact Python ${PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION} not found, trying with manual hints...") - - # Try to manually set paths if pyenv is being used - if(DEFINED ENV{PYENV_VERSION}) - set(Python_EXECUTABLE "$ENV{PYENV_ROOT}/versions/$ENV{PYENV_VERSION}/bin/python") - set(Python_INCLUDE_DIRS "$ENV{PYENV_ROOT}/versions/$ENV{PYENV_VERSION}/include/python${PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION}") - set(Python_LIBRARY_DIRS "$ENV{PYENV_ROOT}/versions/$ENV{PYENV_VERSION}/lib") - - # Try to find the library file - find_library(Python_LIBRARIES - NAMES python${PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION} python${PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION}m - PATHS "${Python_LIBRARY_DIRS}" - NO_DEFAULT_PATH - ) - - if(Python_EXECUTABLE AND Python_INCLUDE_DIRS AND Python_LIBRARIES) - set(Python_FOUND TRUE) - set(Python_VERSION ${PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION}) - string(REPLACE "." ";" VERSION_LIST ${Python_VERSION}) - list(GET VERSION_LIST 0 Python_VERSION_MAJOR) - list(GET VERSION_LIST 1 Python_VERSION_MINOR) - message(STATUS "Manually configured Python ${Python_VERSION}") - message(STATUS " Executable: ${Python_EXECUTABLE}") - message(STATUS " Include dirs: ${Python_INCLUDE_DIRS}") - message(STATUS " Libraries: ${Python_LIBRARIES}") - endif() - endif() - - if(NOT Python_FOUND) - message(FATAL_ERROR "Could NOT find Python ${PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION} development files") - endif() - endif() + find_package(Python ${PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION} EXACT REQUIRED COMPONENTS Interpreter Development) else() find_package(Python REQUIRED COMPONENTS Interpreter Development) endif() @@ -88,7 +47,6 @@ if (APPLE) target_link_options(pybind11_stubs PRIVATE -undefined dynamic_lookup) else() target_link_options(pybind11_stubs PRIVATE -Wl,--unresolved-symbols=ignore-all) - # For musl cross-compilation, add nostartfiles to avoid crtbeginS.o/crtendS.o issues if (USE_MUSL) target_link_options(pybind11_stubs PRIVATE -nostartfiles) endif() @@ -122,7 +80,6 @@ if(Python_FOUND) target_link_options(${PYBIND11_NONLIMITEDAPI_LIBNAME} PRIVATE -undefined dynamic_lookup) else() target_link_options(${PYBIND11_NONLIMITEDAPI_LIBNAME} PRIVATE -Wl,--unresolved-symbols=ignore-all) - # For musl cross-compilation, add nostartfiles to avoid crtbeginS.o/crtendS.o issues if (USE_MUSL) target_link_options(${PYBIND11_NONLIMITEDAPI_LIBNAME} PRIVATE -nostartfiles) endif() diff --git a/src/Access/Common/AccessFlags.cpp b/src/Access/Common/AccessFlags.cpp index 3d3fe2d3732..9d35bac958e 100644 --- a/src/Access/Common/AccessFlags.cpp +++ b/src/Access/Common/AccessFlags.cpp @@ -266,15 +266,8 @@ namespace start_node->aliases.emplace_back(start_node->keyword); for (auto & alias : start_node->aliases) { - // boost::to_upper(alias); - // keyword_to_flags_map[alias] = start_node->flags; - std::string upper_alias = alias; - for (char & c : upper_alias) - { - if (c >= 'a' && c <= 'z') - c = c - 'a' + 'A'; - } - keyword_to_flags_map[upper_alias] = start_node->flags; + boost::to_upper(alias); + keyword_to_flags_map[alias] = start_node->flags; } for (auto & child : start_node->children) diff --git a/src/Common/AllocationInterceptors.cpp b/src/Common/AllocationInterceptors.cpp index e49f784797d..52d71efe319 100644 --- a/src/Common/AllocationInterceptors.cpp +++ b/src/Common/AllocationInterceptors.cpp @@ -44,65 +44,6 @@ static struct InitializeJemallocZoneAllocatorForOSX } initializeJemallocZoneAllocatorForOSX; #endif -#if USE_JEMALLOC - -namespace Memory -{ - thread_local bool disable_memory_check = false; -} - -#endif - -ALWAYS_INLINE void * tryNewNonJemallocMemoryConditional(std::size_t size) -{ -#if USE_JEMALLOC - if (likely(!Memory::disable_memory_check)) - { - return Memory::realNewImpl(size); - } -#endif - (void)size; - return nullptr; -} - -ALWAYS_INLINE void * tryNewNonJemallocMemoryConditional(std::size_t size, std::align_val_t align) -{ -#if USE_JEMALLOC - if (likely(!Memory::disable_memory_check)) - { - return Memory::realNewImpl(size, align); - } -#endif - (void)size; - (void)align; - return nullptr; -} - -ALWAYS_INLINE void * tryNewNoExceptNonJemallocMemoryConditional(std::size_t size) -{ -#if USE_JEMALLOC - if (likely(!Memory::disable_memory_check)) - { - return Memory::realNewNoExcept(size); - } -#endif - (void)size; - return nullptr; -} - -ALWAYS_INLINE void * tryNewNoExceptNonJemallocMemoryConditional(std::size_t size, std::align_val_t align) -{ -#if USE_JEMALLOC - if (likely(!Memory::disable_memory_check)) - { - return Memory::realNewNoExcept(size, align); - } -#endif - (void)size; - (void)align; - return nullptr; -} - /// Replace default new/delete with memory tracking versions. /// @sa https://en.cppreference.com/w/cpp/memory/new/operator_new /// https://en.cppreference.com/w/cpp/memory/new/operator_delete @@ -111,9 +52,6 @@ ALWAYS_INLINE void * tryNewNoExceptNonJemallocMemoryConditional(std::size_t size void * operator new(std::size_t size) { - if (void * ptr = tryNewNonJemallocMemoryConditional(size)) - return ptr; - AllocationTrace trace; std::size_t actual_size = Memory::trackMemory(size, trace); void * ptr = Memory::newImpl(size); @@ -123,9 +61,6 @@ void * operator new(std::size_t size) void * operator new(std::size_t size, std::align_val_t align) { - if (void * ptr = tryNewNonJemallocMemoryConditional(size, align)) - return ptr; - AllocationTrace trace; std::size_t actual_size = Memory::trackMemory(size, trace, align); void * ptr = Memory::newImpl(size, align); @@ -135,9 +70,6 @@ void * operator new(std::size_t size, std::align_val_t align) void * operator new[](std::size_t size) { - if (void * ptr = tryNewNonJemallocMemoryConditional(size)) - return ptr; - AllocationTrace trace; std::size_t actual_size = Memory::trackMemory(size, trace); void * ptr = Memory::newImpl(size); @@ -147,9 +79,6 @@ void * operator new[](std::size_t size) void * operator new[](std::size_t size, std::align_val_t align) { - if (void * ptr = tryNewNonJemallocMemoryConditional(size, align)) - return ptr; - AllocationTrace trace; std::size_t actual_size = Memory::trackMemory(size, trace, align); void * ptr = Memory::newImpl(size, align); @@ -159,9 +88,6 @@ void * operator new[](std::size_t size, std::align_val_t align) void * operator new(std::size_t size, const std::nothrow_t &) noexcept { - if (void * ptr = tryNewNoExceptNonJemallocMemoryConditional(size)) - return ptr; - AllocationTrace trace; std::size_t actual_size = Memory::trackMemory(size, trace); void * ptr = Memory::newNoExcept(size); @@ -171,9 +97,6 @@ void * operator new(std::size_t size, const std::nothrow_t &) noexcept void * operator new[](std::size_t size, const std::nothrow_t &) noexcept { - if (void * ptr = tryNewNoExceptNonJemallocMemoryConditional(size)) - return ptr; - AllocationTrace trace; std::size_t actual_size = Memory::trackMemory(size, trace); void * ptr = Memory::newNoExcept(size); @@ -183,9 +106,6 @@ void * operator new[](std::size_t size, const std::nothrow_t &) noexcept void * operator new(std::size_t size, std::align_val_t align, const std::nothrow_t &) noexcept { - if (void * ptr = tryNewNoExceptNonJemallocMemoryConditional(size, align)) - return ptr; - AllocationTrace trace; std::size_t actual_size = Memory::trackMemory(size, trace, align); void * ptr = Memory::newNoExcept(size, align); @@ -195,9 +115,6 @@ void * operator new(std::size_t size, std::align_val_t align, const std::nothrow void * operator new[](std::size_t size, std::align_val_t align, const std::nothrow_t &) noexcept { - if (void * ptr = tryNewNoExceptNonJemallocMemoryConditional(size, align)) - return ptr; - AllocationTrace trace; std::size_t actual_size = Memory::trackMemory(size, trace, align); void * ptr = Memory::newNoExcept(size, align); @@ -209,7 +126,7 @@ void * operator new[](std::size_t size, std::align_val_t align, const std::nothr extern "C" void __real_free(void * ptr); -ALWAYS_INLINE bool isJemallocMemory(void * ptr) +inline ALWAYS_INLINE bool isJemallocMemory(void * ptr) { int arena_ind = je_mallctl("arenas.lookup", nullptr, nullptr, &ptr, sizeof(ptr)); return arena_ind == 0; // arena_ind == 0 means jemalloc memory @@ -228,7 +145,7 @@ ALWAYS_INLINE bool isJemallocMemory(void * ptr) /// /// Note: We don't update memory tracking for non-jemalloc memory since it was likely /// never tracked by our system in the first place. -ALWAYS_INLINE bool tryFreeNonJemallocMemory(void * ptr) +inline ALWAYS_INLINE bool tryFreeNonJemallocMemory(void * ptr) { if (unlikely(ptr == nullptr)) return true; @@ -243,7 +160,12 @@ ALWAYS_INLINE bool tryFreeNonJemallocMemory(void * ptr) return false; // Not handled - should continue with jemalloc path } -ALWAYS_INLINE bool tryFreeNonJemallocMemoryConditional(void * ptr) +namespace Memory +{ +thread_local bool disable_memory_check{false}; +} + +inline ALWAYS_INLINE bool tryFreeNonJemallocMemoryConditional(void * ptr) { if (unlikely(ptr == nullptr)) return true; diff --git a/src/Common/memory.h b/src/Common/memory.h index 143a4e34182..cbe9f7b7c64 100644 --- a/src/Common/memory.h +++ b/src/Common/memory.h @@ -87,23 +87,6 @@ inline ALWAYS_INLINE void * newImpl(std::size_t size, TAlign... align) throw std::bad_alloc{}; } -template ... TAlign> -requires DB::OptionalArgument -inline ALWAYS_INLINE void * realNewImpl(std::size_t size, TAlign... align) -{ - void * ptr = nullptr; - if constexpr (sizeof...(TAlign) == 1) - ptr = __real_aligned_alloc(alignToSizeT(align...), alignUp(size, alignToSizeT(align...))); - else - ptr = __real_malloc(size); - - if (likely(ptr != nullptr)) - return ptr; - - /// @note no std::get_new_handler logic implemented - throw std::bad_alloc{}; -} - #else template ... TAlign> requires DB::OptionalArgument @@ -139,16 +122,6 @@ inline ALWAYS_INLINE void deleteImpl(void * ptr) noexcept je_free(ptr); } -inline ALWAYS_INLINE void * realNewNoExcept(std::size_t size) noexcept -{ - return __real_malloc(size); -} - -inline ALWAYS_INLINE void * realNewNoExcept(std::size_t size, std::align_val_t align) noexcept -{ - return __real_aligned_alloc(static_cast(align), size); -} - #else inline ALWAYS_INLINE void * newNoExcept(std::size_t size) noexcept From a9658a1ee3c09036b58117b6d572352a9f82aaca Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Mon, 13 Oct 2025 21:56:02 +0800 Subject: [PATCH 03/41] chore: update AllocationInterceptors.h --- src/Common/AllocationInterceptors.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/AllocationInterceptors.h b/src/Common/AllocationInterceptors.h index 55b364f2643..d5606037b46 100644 --- a/src/Common/AllocationInterceptors.h +++ b/src/Common/AllocationInterceptors.h @@ -8,7 +8,7 @@ // NOLINTBEGIN -#if defined(SANITIZER) || defined(SANITIZE_COVERAGE) || defined(OS_DARWIN) || defined(OS_FREEBSD) || !USE_JEMALLOC +#if defined(SANITIZER) || defined(SANITIZE_COVERAGE) || defined(OS_DARWIN) || defined(OS_FREEBSD) || !USE_JEMALLOC || !defined(USE_MUSL) #define __real_malloc(size) ::malloc(size) #define __real_calloc(nmemb, size) ::calloc(nmemb, size) From 163e09ae81bc6f812f5961efd0586031952ea2fb Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Mon, 13 Oct 2025 22:08:47 +0800 Subject: [PATCH 04/41] chore: update AllocationInterceptors.h --- src/Common/AllocationInterceptors.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/AllocationInterceptors.h b/src/Common/AllocationInterceptors.h index d5606037b46..db16aa2bd46 100644 --- a/src/Common/AllocationInterceptors.h +++ b/src/Common/AllocationInterceptors.h @@ -8,7 +8,7 @@ // NOLINTBEGIN -#if defined(SANITIZER) || defined(SANITIZE_COVERAGE) || defined(OS_DARWIN) || defined(OS_FREEBSD) || !USE_JEMALLOC || !defined(USE_MUSL) +#if defined(SANITIZER) || defined(SANITIZE_COVERAGE) || defined(OS_DARWIN) || defined(OS_FREEBSD) || (!USE_JEMALLOC && !defined(USE_MUSL)) #define __real_malloc(size) ::malloc(size) #define __real_calloc(nmemb, size) ::calloc(nmemb, size) From 3d0e8613c91d066931946e9c2fd6f987ef816c83 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Mon, 13 Oct 2025 22:33:48 +0800 Subject: [PATCH 05/41] chore: update workflow --- .../workflows/build_linux_arm64_wheels-gh.yml | 638 +++++++++--------- .github/workflows/build_linux_x86_wheels.yml | 636 ++++++++--------- .../workflows/build_macos_arm64_wheels.yml | 544 +++++++-------- .github/workflows/build_macos_x86_wheels.yml | 588 ++++++++-------- .../workflows/build_musllinux_x86_wheels.yml | 265 ++++++++ chdb/vars.sh | 3 +- setup.py | 3 +- 7 files changed, 1470 insertions(+), 1207 deletions(-) create mode 100644 .github/workflows/build_musllinux_x86_wheels.yml diff --git a/.github/workflows/build_linux_arm64_wheels-gh.yml b/.github/workflows/build_linux_arm64_wheels-gh.yml index fe9e78277a5..4f677959e52 100644 --- a/.github/workflows/build_linux_arm64_wheels-gh.yml +++ b/.github/workflows/build_linux_arm64_wheels-gh.yml @@ -1,333 +1,333 @@ -name: Build Linux arm64 +# name: Build Linux arm64 -on: - workflow_dispatch: - inputs: - TAG_NAME: - description: 'Release Version Tag' - required: true - release: - types: [created] - push: - branches: - - main - paths-ignore: - - '**/*.md' - pull_request: - branches: - - main - paths-ignore: - - '**/*.md' +# on: +# workflow_dispatch: +# inputs: +# TAG_NAME: +# description: 'Release Version Tag' +# required: true +# release: +# types: [created] +# push: +# branches: +# - main +# paths-ignore: +# - '**/*.md' +# pull_request: +# branches: +# - main +# paths-ignore: +# - '**/*.md' -jobs: - build_universal_wheel: - name: Build Universal Wheel (Linux ARM64) - runs-on: GH-Linux-ARM64 - steps: - - name: Install Python build dependencies - run: | - sudo apt-get update - sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \ - libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ - libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \ - libffi-dev liblzma-dev - - name: Scan SQLite vulnerabilities with grype - run: | - # Install grype and required tools - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin - sudo apt-get update && sudo apt-get install -y jq lsb-release +# jobs: +# build_universal_wheel: +# name: Build Universal Wheel (Linux ARM64) +# runs-on: GH-Linux-ARM64 +# steps: +# - name: Install Python build dependencies +# run: | +# sudo apt-get update +# sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \ +# libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ +# libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \ +# libffi-dev liblzma-dev +# - name: Scan SQLite vulnerabilities with grype +# run: | +# # Install grype and required tools +# curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin +# sudo apt-get update && sudo apt-get install -y jq lsb-release - # Detect OS distribution info - DISTRO_ID=$(lsb_release -si | tr '[:upper:]' '[:lower:]') - DISTRO_VERSION=$(lsb_release -sr) - echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION" +# # Detect OS distribution info +# DISTRO_ID=$(lsb_release -si | tr '[:upper:]' '[:lower:]') +# DISTRO_VERSION=$(lsb_release -sr) +# echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION" - # Update grype vulnerability database - grype db update +# # Update grype vulnerability database +# grype db update - # Check SQLite vulnerabilities in installed packages - echo "Scanning SQLite packages for vulnerabilities..." - GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true) - echo "Raw grype output:" - echo "$GRYPE_RAW_OUTPUT" +# # Check SQLite vulnerabilities in installed packages +# echo "Scanning SQLite packages for vulnerabilities..." +# GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true) +# echo "Raw grype output:" +# echo "$GRYPE_RAW_OUTPUT" - SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) +# SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) - if [ -n "$SQLITE_SCAN_OUTPUT" ]; then - echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." - echo "SQLite vulnerability details:" - echo "$SQLITE_SCAN_OUTPUT" - exit 1 - else - echo "✅ No SQLite vulnerabilities found" - fi - continue-on-error: false - - name: Setup pyenv - run: | - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv install 3.8:latest - pyenv install 3.9:latest - pyenv install 3.10:latest - pyenv install 3.11:latest - pyenv install 3.12:latest - pyenv install 3.13:latest - pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 +# if [ -n "$SQLITE_SCAN_OUTPUT" ]; then +# echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." +# echo "SQLite vulnerability details:" +# echo "$SQLITE_SCAN_OUTPUT" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found" +# fi +# continue-on-error: false +# - name: Setup pyenv +# run: | +# curl https://pyenv.run | bash +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv install 3.8:latest +# pyenv install 3.9:latest +# pyenv install 3.10:latest +# pyenv install 3.11:latest +# pyenv install 3.12:latest +# pyenv install 3.13:latest +# pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 - # Verify installations - echo "Installed versions:" - pyenv versions - - name: Verify pyenv installations - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - echo "Installed Python versions:" - pyenv versions - echo "" - echo "Verifying all required Python versions are available:" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - if ! pyenv versions --bare | grep -q "^$version"; then - echo "ERROR: Python $version is not installed!" - exit 1 - fi - echo "✓ Python $version is installed" - done - echo "All Python versions verified successfully!" - - name: Install dependencies for all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - echo "Installing dependencies for Python $version" - pyenv shell $version - python -m pip install --upgrade pip - python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel - pyenv shell --unset - done - - name: Upgrade Rust toolchain - run: | - rustup toolchain install nightly-2025-07-07 - rustup default nightly-2025-07-07 - rustup component add rust-src - rustc --version - cargo --version - - name: Install clang++ for Ubuntu - run: | - pwd - uname -a - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh 19 - which clang++-19 - clang++-19 --version - sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget - # Install WebAssembly linker (wasm-ld) - sudo apt-get install -y lld-19 - # Create symlink for wasm-ld - if ! command -v wasm-ld &> /dev/null; then - sudo ln -sf /usr/bin/wasm-ld-19 /usr/bin/wasm-ld || true - fi - which wasm-ld || echo "wasm-ld not found in PATH" - ccache -s - - name: Update git - run: | - sudo add-apt-repository ppa:git-core/ppa -y - sudo apt-get update - sudo apt-get install -y git - git --version - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Update submodules - run: | - git submodule update --init --recursive --jobs 4 - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: ubuntu-24.04-aarch64 - max-size: 5G - append-timestamp: true - - name: remove old clang and link clang-19 to clang - run: | - sudo rm -f /usr/bin/clang || true - sudo ln -s /usr/bin/clang-19 /usr/bin/clang - sudo rm -f /usr/bin/clang++ || true - sudo ln -s /usr/bin/clang++-19 /usr/bin/clang++ - which clang++ - clang++ --version - - name: Run chdb/build.sh - timeout-minutes: 600 - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - source ~/.cargo/env - pyenv shell 3.8 - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - bash ./chdb/build.sh - pyenv shell 3.8 - bash -x ./chdb/test_smoke.sh - continue-on-error: false - - name: Run chdb/build/build_static_lib.sh - timeout-minutes: 600 - run: | - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - export PATH="$HOME/.pyenv/bin:$PATH" - source ~/.cargo/env - eval "$(pyenv init -)" - pyenv shell 3.8 - bash ./chdb/build/build_static_lib.sh - pyenv shell --unset - continue-on-error: false - - name: Scan chdb libraries with grype - run: | - echo "Scanning chdb libraries for vulnerabilities..." +# # Verify installations +# echo "Installed versions:" +# pyenv versions +# - name: Verify pyenv installations +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# echo "Installed Python versions:" +# pyenv versions +# echo "" +# echo "Verifying all required Python versions are available:" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# if ! pyenv versions --bare | grep -q "^$version"; then +# echo "ERROR: Python $version is not installed!" +# exit 1 +# fi +# echo "✓ Python $version is installed" +# done +# echo "All Python versions verified successfully!" +# - name: Install dependencies for all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# echo "Installing dependencies for Python $version" +# pyenv shell $version +# python -m pip install --upgrade pip +# python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel +# pyenv shell --unset +# done +# - name: Upgrade Rust toolchain +# run: | +# rustup toolchain install nightly-2025-07-07 +# rustup default nightly-2025-07-07 +# rustup component add rust-src +# rustc --version +# cargo --version +# - name: Install clang++ for Ubuntu +# run: | +# pwd +# uname -a +# wget https://apt.llvm.org/llvm.sh +# chmod +x llvm.sh +# sudo ./llvm.sh 19 +# which clang++-19 +# clang++-19 --version +# sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget +# # Install WebAssembly linker (wasm-ld) +# sudo apt-get install -y lld-19 +# # Create symlink for wasm-ld +# if ! command -v wasm-ld &> /dev/null; then +# sudo ln -sf /usr/bin/wasm-ld-19 /usr/bin/wasm-ld || true +# fi +# which wasm-ld || echo "wasm-ld not found in PATH" +# ccache -s +# - name: Update git +# run: | +# sudo add-apt-repository ppa:git-core/ppa -y +# sudo apt-get update +# sudo apt-get install -y git +# git --version +# - uses: actions/checkout@v3 +# with: +# fetch-depth: 0 +# - name: Update submodules +# run: | +# git submodule update --init --recursive --jobs 4 +# - name: ccache +# uses: hendrikmuhs/ccache-action@v1.2 +# with: +# key: ubuntu-24.04-aarch64 +# max-size: 5G +# append-timestamp: true +# - name: remove old clang and link clang-19 to clang +# run: | +# sudo rm -f /usr/bin/clang || true +# sudo ln -s /usr/bin/clang-19 /usr/bin/clang +# sudo rm -f /usr/bin/clang++ || true +# sudo ln -s /usr/bin/clang++-19 /usr/bin/clang++ +# which clang++ +# clang++ --version +# - name: Run chdb/build.sh +# timeout-minutes: 600 +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# source ~/.cargo/env +# pyenv shell 3.8 +# export CC=/usr/bin/clang +# export CXX=/usr/bin/clang++ +# bash ./chdb/build.sh +# pyenv shell 3.8 +# bash -x ./chdb/test_smoke.sh +# continue-on-error: false +# - name: Run chdb/build/build_static_lib.sh +# timeout-minutes: 600 +# run: | +# export CC=/usr/bin/clang +# export CXX=/usr/bin/clang++ +# export PATH="$HOME/.pyenv/bin:$PATH" +# source ~/.cargo/env +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# bash ./chdb/build/build_static_lib.sh +# pyenv shell --unset +# continue-on-error: false +# - name: Scan chdb libraries with grype +# run: | +# echo "Scanning chdb libraries for vulnerabilities..." - # Files to scan - FILES_TO_SCAN="" - [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" - [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" - FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" +# # Files to scan +# FILES_TO_SCAN="" +# [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" +# [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" +# FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" - SQLITE_VULNERABILITIES_FOUND=false +# SQLITE_VULNERABILITIES_FOUND=false - for file in $FILES_TO_SCAN; do - if [ -f "$file" ]; then - echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) - echo "$SCAN_OUTPUT" +# for file in $FILES_TO_SCAN; do +# if [ -f "$file" ]; then +# echo "=== Scanning $file ===" +# SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) +# echo "$SCAN_OUTPUT" - if echo "$SCAN_OUTPUT" | grep -qi sqlite; then - echo "❌ SQLite vulnerability found in $file" - SQLITE_VULNERABILITIES_FOUND=true - fi - fi - done +# if echo "$SCAN_OUTPUT" | grep -qi sqlite; then +# echo "❌ SQLite vulnerability found in $file" +# SQLITE_VULNERABILITIES_FOUND=true +# fi +# fi +# done - if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then - echo "❌ SQLite vulnerabilities detected in chdb libraries!" - exit 1 - else - echo "✅ No SQLite vulnerabilities found in chdb libraries" - fi - continue-on-error: false - - name: Run libchdb stub in examples dir - run: | - bash -x ./examples/runStub.sh - - name: Check ccache statistics - run: | - ccache -s - ls -lh chdb - df -h - - name: Build wheels - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - pyenv shell 3.8 - make wheel - - name: Install patchelf from github - run: | - wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz - tar -xvf patchelf.tar.gz - sudo cp bin/patchelf /usr/bin/ - sudo chmod +x /usr/bin/patchelf - patchelf --version - - name: Audit wheels - run: | - python3 -m pip install auditwheel - auditwheel -v repair -w dist/ --plat manylinux_2_17_aarch64 dist/*.whl - continue-on-error: false - - name: Show files - run: | - # e.g: remove chdb-0.11.4-cp310-cp310-linux_aarch64.whl, keep chdb-0.11.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl - sudo rm -f dist/*linux_aarch64.whl - ls -lh dist - shell: bash - - name: Setup core dump collection - run: | - mkdir -p tmp/core - echo "tmp/core/core.%p" | sudo tee /proc/sys/kernel/core_pattern - ulimit -c unlimited - - name: Test wheel on all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - echo "Testing chdb on Python $version" - pyenv shell $version - python -m pip install dist/*.whl --force-reinstall - python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" - make test - pyenv shell --unset - done - continue-on-error: false - - name: Check and upload core files if present - if: always() - run: | - if ls tmp/core/core.* >/dev/null 2>&1; then - echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV - tar -czvf core-files-linux-aarch64.tar.gz tmp/core/core.* - echo "Core files tar created: core-files-linux-aarch64.tar.gz" - ls -lh core-files-linux-aarch64.tar.gz - else - echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV - echo "No core files found in tmp/core" - fi - continue-on-error: true - - name: Upload core files if present - if: always() && env.CORE_FILES_FOUND == 'true' - uses: actions/upload-artifact@v4 - with: - name: core-files-linux-aarch64 - path: core-files-linux-aarch64.tar.gz - - name: Upload wheels to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} dist/*.whl --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Packege libchdb.so - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf linux-aarch64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp - - name: Package libchdb.a - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf linux-aarch64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp - - name: Upload libchdb.so to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} linux-aarch64-libchdb.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Upload libchdb.a to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} linux-aarch64-libchdb-static.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - uses: actions/upload-artifact@v4 - with: - name: chdb-artifacts-linux-aarch64 - path: | - ./dist/*.whl - ./linux-aarch64-libchdb.tar.gz - ./linux-aarch64-libchdb-static.tar.gz - overwrite: true - - name: Upload pypi - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - python -m twine upload dist/*.whl - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} +# if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then +# echo "❌ SQLite vulnerabilities detected in chdb libraries!" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found in chdb libraries" +# fi +# continue-on-error: false +# - name: Run libchdb stub in examples dir +# run: | +# bash -x ./examples/runStub.sh +# - name: Check ccache statistics +# run: | +# ccache -s +# ls -lh chdb +# df -h +# - name: Build wheels +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# export CC=/usr/bin/clang +# export CXX=/usr/bin/clang++ +# pyenv shell 3.8 +# make wheel +# - name: Install patchelf from github +# run: | +# wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz +# tar -xvf patchelf.tar.gz +# sudo cp bin/patchelf /usr/bin/ +# sudo chmod +x /usr/bin/patchelf +# patchelf --version +# - name: Audit wheels +# run: | +# python3 -m pip install auditwheel +# auditwheel -v repair -w dist/ --plat manylinux_2_17_aarch64 dist/*.whl +# continue-on-error: false +# - name: Show files +# run: | +# # e.g: remove chdb-0.11.4-cp310-cp310-linux_aarch64.whl, keep chdb-0.11.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl +# sudo rm -f dist/*linux_aarch64.whl +# ls -lh dist +# shell: bash +# - name: Setup core dump collection +# run: | +# mkdir -p tmp/core +# echo "tmp/core/core.%p" | sudo tee /proc/sys/kernel/core_pattern +# ulimit -c unlimited +# - name: Test wheel on all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# echo "Testing chdb on Python $version" +# pyenv shell $version +# python -m pip install dist/*.whl --force-reinstall +# python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" +# make test +# pyenv shell --unset +# done +# continue-on-error: false +# - name: Check and upload core files if present +# if: always() +# run: | +# if ls tmp/core/core.* >/dev/null 2>&1; then +# echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV +# tar -czvf core-files-linux-aarch64.tar.gz tmp/core/core.* +# echo "Core files tar created: core-files-linux-aarch64.tar.gz" +# ls -lh core-files-linux-aarch64.tar.gz +# else +# echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV +# echo "No core files found in tmp/core" +# fi +# continue-on-error: true +# - name: Upload core files if present +# if: always() && env.CORE_FILES_FOUND == 'true' +# uses: actions/upload-artifact@v4 +# with: +# name: core-files-linux-aarch64 +# path: core-files-linux-aarch64.tar.gz +# - name: Upload wheels to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} dist/*.whl --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Packege libchdb.so +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf linux-aarch64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp +# - name: Package libchdb.a +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf linux-aarch64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp +# - name: Upload libchdb.so to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} linux-aarch64-libchdb.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Upload libchdb.a to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} linux-aarch64-libchdb-static.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - uses: actions/upload-artifact@v4 +# with: +# name: chdb-artifacts-linux-aarch64 +# path: | +# ./dist/*.whl +# ./linux-aarch64-libchdb.tar.gz +# ./linux-aarch64-libchdb-static.tar.gz +# overwrite: true +# - name: Upload pypi +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# python -m twine upload dist/*.whl +# env: +# TWINE_USERNAME: __token__ +# TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/build_linux_x86_wheels.yml b/.github/workflows/build_linux_x86_wheels.yml index 5a35668ead9..29a71bbb668 100644 --- a/.github/workflows/build_linux_x86_wheels.yml +++ b/.github/workflows/build_linux_x86_wheels.yml @@ -1,332 +1,332 @@ -name: Build Linux X86 +# name: Build Linux X86 -on: - workflow_dispatch: - inputs: - TAG_NAME: - description: 'Release Version Tag' - required: true - release: - types: [created] - push: - branches: - - main - paths-ignore: - - '**/*.md' - pull_request: - branches: - - main - paths-ignore: - - '**/*.md' +# on: +# workflow_dispatch: +# inputs: +# TAG_NAME: +# description: 'Release Version Tag' +# required: true +# release: +# types: [created] +# push: +# branches: +# - main +# paths-ignore: +# - '**/*.md' +# pull_request: +# branches: +# - main +# paths-ignore: +# - '**/*.md' -jobs: - build_universal_wheel: - name: Build Universal Wheel (Linux x86_64) - runs-on: gh-64c - steps: - - name: Install Python build dependencies - run: | - sudo apt-get update - sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \ - libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ - libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \ - libffi-dev liblzma-dev - - name: Scan SQLite vulnerabilities with grype - run: | - # Install grype and required tools - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin - sudo apt-get update && sudo apt-get install -y jq lsb-release +# jobs: +# build_universal_wheel: +# name: Build Universal Wheel (Linux x86_64) +# runs-on: gh-64c +# steps: +# - name: Install Python build dependencies +# run: | +# sudo apt-get update +# sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \ +# libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ +# libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \ +# libffi-dev liblzma-dev +# - name: Scan SQLite vulnerabilities with grype +# run: | +# # Install grype and required tools +# curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin +# sudo apt-get update && sudo apt-get install -y jq lsb-release - # Detect OS distribution info - DISTRO_ID=$(lsb_release -si | tr '[:upper:]' '[:lower:]') - DISTRO_VERSION=$(lsb_release -sr) - echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION" +# # Detect OS distribution info +# DISTRO_ID=$(lsb_release -si | tr '[:upper:]' '[:lower:]') +# DISTRO_VERSION=$(lsb_release -sr) +# echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION" - # Update grype vulnerability database - grype db update +# # Update grype vulnerability database +# grype db update - # Check SQLite vulnerabilities in installed packages - echo "Scanning SQLite packages for vulnerabilities..." - GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true) - echo "Raw grype output:" - echo "$GRYPE_RAW_OUTPUT" +# # Check SQLite vulnerabilities in installed packages +# echo "Scanning SQLite packages for vulnerabilities..." +# GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true) +# echo "Raw grype output:" +# echo "$GRYPE_RAW_OUTPUT" - SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) +# SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) - if [ -n "$SQLITE_SCAN_OUTPUT" ]; then - echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." - echo "SQLite vulnerability details:" - echo "$SQLITE_SCAN_OUTPUT" - exit 1 - else - echo "✅ No SQLite vulnerabilities found" - fi - continue-on-error: false - - name: Setup pyenv - run: | - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv install 3.8:latest - pyenv install 3.9:latest - pyenv install 3.10:latest - pyenv install 3.11:latest - pyenv install 3.12:latest - pyenv install 3.13:latest - pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 +# if [ -n "$SQLITE_SCAN_OUTPUT" ]; then +# echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." +# echo "SQLite vulnerability details:" +# echo "$SQLITE_SCAN_OUTPUT" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found" +# fi +# continue-on-error: false +# - name: Setup pyenv +# run: | +# curl https://pyenv.run | bash +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv install 3.8:latest +# pyenv install 3.9:latest +# pyenv install 3.10:latest +# pyenv install 3.11:latest +# pyenv install 3.12:latest +# pyenv install 3.13:latest +# pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 - # Verify installations - echo "Installed versions:" - pyenv versions - - name: Verify pyenv installations - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - echo "Installed Python versions:" - pyenv versions - echo "" - echo "Verifying all required Python versions are available:" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - if ! pyenv versions --bare | grep -q "^$version"; then - echo "ERROR: Python $version is not installed!" - exit 1 - fi - echo "✓ Python $version is installed" - done - echo "All Python versions verified successfully!" - - name: Install dependencies for all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - echo "Installing dependencies for Python $version" - pyenv shell $version - python -m pip install --upgrade pip - python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel - pyenv shell --unset - done - - name: Upgrade Rust toolchain - run: | - rustup toolchain install nightly-2025-07-07 - rustup default nightly-2025-07-07 - rustup component add rust-src - rustc --version - cargo --version - - name: Install clang++ for Ubuntu - run: | - pwd - uname -a - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh 19 - which clang++-19 - clang++-19 --version - sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget - # Install WebAssembly linker (wasm-ld) - sudo apt-get install -y lld-19 - # Create symlink for wasm-ld - if ! command -v wasm-ld &> /dev/null; then - sudo ln -sf /usr/bin/wasm-ld-19 /usr/bin/wasm-ld || true - fi - which wasm-ld || echo "wasm-ld not found in PATH" - ccache -s - - name: Update git - run: | - sudo add-apt-repository ppa:git-core/ppa -y - sudo apt-get update - sudo apt-get install -y git - git --version - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Update submodules - run: | - git submodule update --init --recursive --jobs 4 - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: ubuntu-22.04-x86_64 - max-size: 5G - append-timestamp: true - - name: remove old clang and link clang-19 to clang - run: | - sudo rm -f /usr/bin/clang || true - sudo ln -s /usr/bin/clang-19 /usr/bin/clang - sudo rm -f /usr/bin/clang++ || true - sudo ln -s /usr/bin/clang++-19 /usr/bin/clang++ - which clang++ - clang++ --version - - name: Run chdb/build.sh - timeout-minutes: 600 - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - source ~/.cargo/env - pyenv shell 3.8 - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - bash ./chdb/build.sh - pyenv shell 3.8 - bash -x ./chdb/test_smoke.sh - continue-on-error: false - - name: Run chdb/build/build_static_lib.sh - timeout-minutes: 600 - run: | - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - export PATH="$HOME/.pyenv/bin:$PATH" - source ~/.cargo/env - eval "$(pyenv init -)" - pyenv shell 3.8 - bash ./chdb/build/build_static_lib.sh - pyenv shell --unset - continue-on-error: false - - name: Scan chdb libraries with grype - run: | - echo "Scanning chdb libraries for vulnerabilities..." +# # Verify installations +# echo "Installed versions:" +# pyenv versions +# - name: Verify pyenv installations +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# echo "Installed Python versions:" +# pyenv versions +# echo "" +# echo "Verifying all required Python versions are available:" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# if ! pyenv versions --bare | grep -q "^$version"; then +# echo "ERROR: Python $version is not installed!" +# exit 1 +# fi +# echo "✓ Python $version is installed" +# done +# echo "All Python versions verified successfully!" +# - name: Install dependencies for all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# echo "Installing dependencies for Python $version" +# pyenv shell $version +# python -m pip install --upgrade pip +# python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel +# pyenv shell --unset +# done +# - name: Upgrade Rust toolchain +# run: | +# rustup toolchain install nightly-2025-07-07 +# rustup default nightly-2025-07-07 +# rustup component add rust-src +# rustc --version +# cargo --version +# - name: Install clang++ for Ubuntu +# run: | +# pwd +# uname -a +# wget https://apt.llvm.org/llvm.sh +# chmod +x llvm.sh +# sudo ./llvm.sh 19 +# which clang++-19 +# clang++-19 --version +# sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget +# # Install WebAssembly linker (wasm-ld) +# sudo apt-get install -y lld-19 +# # Create symlink for wasm-ld +# if ! command -v wasm-ld &> /dev/null; then +# sudo ln -sf /usr/bin/wasm-ld-19 /usr/bin/wasm-ld || true +# fi +# which wasm-ld || echo "wasm-ld not found in PATH" +# ccache -s +# - name: Update git +# run: | +# sudo add-apt-repository ppa:git-core/ppa -y +# sudo apt-get update +# sudo apt-get install -y git +# git --version +# - uses: actions/checkout@v3 +# with: +# fetch-depth: 0 +# - name: Update submodules +# run: | +# git submodule update --init --recursive --jobs 4 +# - name: ccache +# uses: hendrikmuhs/ccache-action@v1.2 +# with: +# key: ubuntu-22.04-x86_64 +# max-size: 5G +# append-timestamp: true +# - name: remove old clang and link clang-19 to clang +# run: | +# sudo rm -f /usr/bin/clang || true +# sudo ln -s /usr/bin/clang-19 /usr/bin/clang +# sudo rm -f /usr/bin/clang++ || true +# sudo ln -s /usr/bin/clang++-19 /usr/bin/clang++ +# which clang++ +# clang++ --version +# - name: Run chdb/build.sh +# timeout-minutes: 600 +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# source ~/.cargo/env +# pyenv shell 3.8 +# export CC=/usr/bin/clang +# export CXX=/usr/bin/clang++ +# bash ./chdb/build.sh +# pyenv shell 3.8 +# bash -x ./chdb/test_smoke.sh +# continue-on-error: false +# - name: Run chdb/build/build_static_lib.sh +# timeout-minutes: 600 +# run: | +# export CC=/usr/bin/clang +# export CXX=/usr/bin/clang++ +# export PATH="$HOME/.pyenv/bin:$PATH" +# source ~/.cargo/env +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# bash ./chdb/build/build_static_lib.sh +# pyenv shell --unset +# continue-on-error: false +# - name: Scan chdb libraries with grype +# run: | +# echo "Scanning chdb libraries for vulnerabilities..." - # Files to scan - FILES_TO_SCAN="" - [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" - [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" - FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" +# # Files to scan +# FILES_TO_SCAN="" +# [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" +# [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" +# FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" - SQLITE_VULNERABILITIES_FOUND=false +# SQLITE_VULNERABILITIES_FOUND=false - for file in $FILES_TO_SCAN; do - if [ -f "$file" ]; then - echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) - echo "$SCAN_OUTPUT" +# for file in $FILES_TO_SCAN; do +# if [ -f "$file" ]; then +# echo "=== Scanning $file ===" +# SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) +# echo "$SCAN_OUTPUT" - if echo "$SCAN_OUTPUT" | grep -qi sqlite; then - echo "❌ SQLite vulnerability found in $file" - SQLITE_VULNERABILITIES_FOUND=true - fi - fi - done +# if echo "$SCAN_OUTPUT" | grep -qi sqlite; then +# echo "❌ SQLite vulnerability found in $file" +# SQLITE_VULNERABILITIES_FOUND=true +# fi +# fi +# done - if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then - echo "❌ SQLite vulnerabilities detected in chdb libraries!" - exit 1 - else - echo "✅ No SQLite vulnerabilities found in chdb libraries" - fi - continue-on-error: false - - name: Run libchdb stub in examples dir - run: | - bash -x ./examples/runStub.sh - - name: Check ccache statistics - run: | - ccache -s - ls -lh chdb - df -h - - name: Build wheels - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - pyenv shell 3.8 - make wheel - - name: Install patchelf from github - run: | - wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz -O patchelf.tar.gz - tar -xvf patchelf.tar.gz - sudo cp bin/patchelf /usr/bin/ - sudo chmod +x /usr/bin/patchelf - patchelf --version - - name: Audit wheels - run: | - python3 -m pip install auditwheel - auditwheel -v repair -w dist/ --plat manylinux2014_x86_64 dist/*.whl - continue-on-error: false - - name: Show files - run: | - sudo rm -f dist/*-linux_x86_64.whl - ls -lh dist - shell: bash - - name: Setup core dump collection - run: | - mkdir -p tmp/core - echo "tmp/core/core.%p" | sudo tee /proc/sys/kernel/core_pattern - ulimit -c unlimited - - name: Test wheel on all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - echo "Testing chdb on Python $version" - pyenv shell $version - python -m pip install dist/*.whl --force-reinstall - python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" - make test - pyenv shell --unset - done - continue-on-error: false - - name: Check and upload core files if present - if: always() - run: | - if ls tmp/core/core.* >/dev/null 2>&1; then - echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV - tar -czvf core-files-linux-x86_64.tar.gz tmp/core/core.* - echo "Core files tar created: core-files-linux-x86_64.tar.gz" - ls -lh core-files-linux-x86_64.tar.gz - else - echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV - echo "No core files found in tmp/core" - fi - continue-on-error: true - - name: Upload core files artifact - if: always() && env.CORE_FILES_FOUND == 'true' - uses: actions/upload-artifact@v4 - with: - name: core-files-linux-x86_64 - path: core-files-linux-x86_64.tar.gz - - name: Upload wheels to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} dist/*.whl --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Packege libchdb.so - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf linux-x86_64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp - - name: Package libchdb.a - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf linux-x86_64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp - - name: Upload libchdb.so to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} linux-x86_64-libchdb.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Upload libchdb.a to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} linux-x86_64-libchdb-static.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - uses: actions/upload-artifact@v4 - with: - name: chdb-artifacts-linux-x86_64 - path: | - ./dist/*.whl - ./linux-x86_64-libchdb.tar.gz - ./linux-x86_64-libchdb-static.tar.gz - overwrite: true - - name: Upload pypi - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - python -m twine upload dist/*.whl - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} +# if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then +# echo "❌ SQLite vulnerabilities detected in chdb libraries!" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found in chdb libraries" +# fi +# continue-on-error: false +# - name: Run libchdb stub in examples dir +# run: | +# bash -x ./examples/runStub.sh +# - name: Check ccache statistics +# run: | +# ccache -s +# ls -lh chdb +# df -h +# - name: Build wheels +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# export CC=/usr/bin/clang +# export CXX=/usr/bin/clang++ +# pyenv shell 3.8 +# make wheel +# - name: Install patchelf from github +# run: | +# wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz -O patchelf.tar.gz +# tar -xvf patchelf.tar.gz +# sudo cp bin/patchelf /usr/bin/ +# sudo chmod +x /usr/bin/patchelf +# patchelf --version +# - name: Audit wheels +# run: | +# python3 -m pip install auditwheel +# auditwheel -v repair -w dist/ --plat manylinux2014_x86_64 dist/*.whl +# continue-on-error: false +# - name: Show files +# run: | +# sudo rm -f dist/*-linux_x86_64.whl +# ls -lh dist +# shell: bash +# - name: Setup core dump collection +# run: | +# mkdir -p tmp/core +# echo "tmp/core/core.%p" | sudo tee /proc/sys/kernel/core_pattern +# ulimit -c unlimited +# - name: Test wheel on all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# echo "Testing chdb on Python $version" +# pyenv shell $version +# python -m pip install dist/*.whl --force-reinstall +# python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" +# make test +# pyenv shell --unset +# done +# continue-on-error: false +# - name: Check and upload core files if present +# if: always() +# run: | +# if ls tmp/core/core.* >/dev/null 2>&1; then +# echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV +# tar -czvf core-files-linux-x86_64.tar.gz tmp/core/core.* +# echo "Core files tar created: core-files-linux-x86_64.tar.gz" +# ls -lh core-files-linux-x86_64.tar.gz +# else +# echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV +# echo "No core files found in tmp/core" +# fi +# continue-on-error: true +# - name: Upload core files artifact +# if: always() && env.CORE_FILES_FOUND == 'true' +# uses: actions/upload-artifact@v4 +# with: +# name: core-files-linux-x86_64 +# path: core-files-linux-x86_64.tar.gz +# - name: Upload wheels to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} dist/*.whl --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Packege libchdb.so +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf linux-x86_64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp +# - name: Package libchdb.a +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf linux-x86_64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp +# - name: Upload libchdb.so to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} linux-x86_64-libchdb.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Upload libchdb.a to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} linux-x86_64-libchdb-static.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - uses: actions/upload-artifact@v4 +# with: +# name: chdb-artifacts-linux-x86_64 +# path: | +# ./dist/*.whl +# ./linux-x86_64-libchdb.tar.gz +# ./linux-x86_64-libchdb-static.tar.gz +# overwrite: true +# - name: Upload pypi +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# python -m twine upload dist/*.whl +# env: +# TWINE_USERNAME: __token__ +# TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/build_macos_arm64_wheels.yml b/.github/workflows/build_macos_arm64_wheels.yml index 61c895f7601..255867f16b6 100644 --- a/.github/workflows/build_macos_arm64_wheels.yml +++ b/.github/workflows/build_macos_arm64_wheels.yml @@ -1,276 +1,276 @@ -name: Build macOS arm64 +# name: Build macOS arm64 -on: - workflow_dispatch: - inputs: - TAG_NAME: - description: 'Release Version Tag' - required: true - release: - types: [created] - push: - branches: - - main - paths-ignore: - - '**/*.md' - pull_request: - branches: - - main - paths-ignore: - - '**/*.md' +# on: +# workflow_dispatch: +# inputs: +# TAG_NAME: +# description: 'Release Version Tag' +# required: true +# release: +# types: [created] +# push: +# branches: +# - main +# paths-ignore: +# - '**/*.md' +# pull_request: +# branches: +# - main +# paths-ignore: +# - '**/*.md' -jobs: - build_universal_wheel: - name: Build Universal Wheel (macOS ARM64) - runs-on: macos-13-xlarge - steps: - - name: Setup pyenv - run: | - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" +# jobs: +# build_universal_wheel: +# name: Build Universal Wheel (macOS ARM64) +# runs-on: macos-13-xlarge +# steps: +# - name: Setup pyenv +# run: | +# curl https://pyenv.run | bash +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" - pyenv install 3.8:latest - pyenv install 3.9:latest - pyenv install 3.10:latest - pyenv install 3.11:latest - pyenv install 3.12:latest - pyenv install 3.13:latest - pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 +# pyenv install 3.8:latest +# pyenv install 3.9:latest +# pyenv install 3.10:latest +# pyenv install 3.11:latest +# pyenv install 3.12:latest +# pyenv install 3.13:latest +# pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 - echo "Installed versions:" - pyenv versions - - name: Verify pyenv installations - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - echo "Installed Python versions:" - pyenv versions - echo "" - echo "Verifying all required Python versions are available:" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - if ! pyenv versions --bare | grep -q "^$version"; then - echo "ERROR: Python $version is not installed!" - exit 1 - fi - echo "✓ Python $version is installed" - done - echo "All Python versions verified successfully!" - - name: Install dependencies for all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - echo "Installing dependencies for Python $version" - pyenv shell $version - python -m pip install --upgrade pip - python -m pip install setuptools wheel tox pandas pyarrow twine psutil deltalake wheel>=0.40.0 - pyenv shell --unset - done - - name: Remove /usr/local/bin/python3 - run: | - sudo rm -f /usr/local/bin/python3 - - name: Install clang++ for macOS - run: | - pwd - uname -a - export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 - brew update - brew install ca-certificates lz4 mpdecimal readline sqlite xz z3 zstd - brew install openssl@3 || echo "OpenSSL install failed, continuing..." - brew install --ignore-dependencies llvm@19 - brew install git ninja libtool gettext binutils grep findutils nasm lld@19 libiconv - brew install go - cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm - export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:$PATH - which clang++ - clang++ --version - which wasm-ld || echo "wasm-ld not found in PATH" - which go - go version - ccache -s | echo "ccache not available yet" - - name: Upgrade Rust toolchain - run: | - rustup toolchain install nightly-2025-07-07 - rustup default nightly-2025-07-07 - rustup component add rust-src - rustc --version - cargo --version - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Update submodules - run: | - git submodule update --init --recursive --jobs 4 - - name: Run chdb/build.sh - timeout-minutes: 600 - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - source ~/.cargo/env - pyenv shell 3.8 - export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - bash gen_manifest.sh - bash ./chdb/build.sh - pyenv shell 3.8 - bash -x ./chdb/test_smoke.sh - - name: Run chdb/build/build_static_lib.sh - timeout-minutes: 600 - run: | - export PATH=$HOME/.pyenv/bin:$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - source ~/.cargo/env - eval "$(pyenv init -)" - pyenv shell 3.8 - bash ./chdb/build/build_static_lib.sh - pyenv shell --unset - continue-on-error: false - - name: Debug libchdb - run: | - ls -lh - llvm-nm libchdb.so | grep query_stable || true - echo "Global Symbol in libchdb.so:" - llvm-nm -g libchdb.so || true - echo "Global Symbol in libclickhouse-local-chdb.a:" - llvm-nm -g buildlib/programs/local/libclickhouse-local-chdb.a || true - echo "Global Symbol in libclickhouse-local-lib.a:" - llvm-nm -g buildlib/programs/local/libclickhouse-local-lib.a || true - echo "pychdb_cmd.sh:" - cat buildlib/pychdb_cmd.sh - echo "libchdb_cmd.sh:" - cat buildlib/libchdb_cmd.sh - - name: Scan chdb libraries with grype - run: | - echo "Scanning chdb libraries for vulnerabilities..." - # Files to scan - FILES_TO_SCAN="" - [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" - [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" - FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)" - SQLITE_VULNERABILITIES_FOUND=false - for file in $FILES_TO_SCAN; do - if [ -f "$file" ]; then - echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) - echo "$SCAN_OUTPUT" - if echo "$SCAN_OUTPUT" | grep -qi sqlite; then - echo "❌ SQLite vulnerability found in $file" - SQLITE_VULNERABILITIES_FOUND=true - fi - fi - done - if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then - echo "❌ SQLite vulnerabilities detected in chdb libraries!" - exit 1 - else - echo "✅ No SQLite vulnerabilities found in chdb libraries" - fi - continue-on-error: false - - name: Run libchdb stub in examples dir - run: | - bash -x ./examples/runStub.sh - - name: Build wheels - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - pyenv shell 3.8 - make wheel - - name: Fix wheel platform tag - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - python -m wheel tags --platform-tag=macosx_11_0_arm64 --remove dist/*.whl - - name: Setup core dump - run: | - mkdir -p tmp/core - sudo sysctl kern.corefile=$PWD/tmp/core/core.%P - sudo sysctl kern.coredump=1 - ulimit -c unlimited - - name: Test wheel on all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - echo "Testing chdb on Python $version" - pyenv shell $version - python -m pip install dist/*.whl --force-reinstall - python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" - make test - pyenv shell --unset - done - continue-on-error: false - - name: Check and upload core files if present - if: always() - run: | - if ls tmp/core/core.* >/dev/null 2>&1; then - echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV - tar -czvf core-files-macos-arm64.tar.gz tmp/core/core.* - echo "Core files tar created: core-files-macos-arm64.tar.gz" - ls -lh core-files-macos-arm64.tar.gz - else - echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV - echo "No core files found in tmp/core" - fi - continue-on-error: true - - name: Upload core files artifact - if: always() && env.CORE_FILES_FOUND == 'true' - uses: actions/upload-artifact@v4 - with: - name: core-files-macos-arm64 - path: core-files-macos-arm64.tar.gz - - name: Show files - run: ls -lh dist - shell: bash - - name: Upload wheels to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} dist/*.whl --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Packege libchdb.so - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf macos-arm64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp - - name: Package libchdb.a - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf macos-arm64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp - - name: Upload libchdb.so to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} macos-arm64-libchdb.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Upload libchdb.a to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} macos-arm64-libchdb-static.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - uses: actions/upload-artifact@v4 - with: - name: chdb-artifacts-macos-arm64 - path: | - ./dist/*.whl - ./macos-arm64-libchdb.tar.gz - ./macos-arm64-libchdb-static.tar.gz - overwrite: true - - name: Upload pypi - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - python -m twine upload dist/*.whl - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} +# echo "Installed versions:" +# pyenv versions +# - name: Verify pyenv installations +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# echo "Installed Python versions:" +# pyenv versions +# echo "" +# echo "Verifying all required Python versions are available:" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# if ! pyenv versions --bare | grep -q "^$version"; then +# echo "ERROR: Python $version is not installed!" +# exit 1 +# fi +# echo "✓ Python $version is installed" +# done +# echo "All Python versions verified successfully!" +# - name: Install dependencies for all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# echo "Installing dependencies for Python $version" +# pyenv shell $version +# python -m pip install --upgrade pip +# python -m pip install setuptools wheel tox pandas pyarrow twine psutil deltalake wheel>=0.40.0 +# pyenv shell --unset +# done +# - name: Remove /usr/local/bin/python3 +# run: | +# sudo rm -f /usr/local/bin/python3 +# - name: Install clang++ for macOS +# run: | +# pwd +# uname -a +# export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 +# brew update +# brew install ca-certificates lz4 mpdecimal readline sqlite xz z3 zstd +# brew install openssl@3 || echo "OpenSSL install failed, continuing..." +# brew install --ignore-dependencies llvm@19 +# brew install git ninja libtool gettext binutils grep findutils nasm lld@19 libiconv +# brew install go +# cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm +# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:$PATH +# which clang++ +# clang++ --version +# which wasm-ld || echo "wasm-ld not found in PATH" +# which go +# go version +# ccache -s | echo "ccache not available yet" +# - name: Upgrade Rust toolchain +# run: | +# rustup toolchain install nightly-2025-07-07 +# rustup default nightly-2025-07-07 +# rustup component add rust-src +# rustc --version +# cargo --version +# - uses: actions/checkout@v3 +# with: +# fetch-depth: 0 +# - name: Update submodules +# run: | +# git submodule update --init --recursive --jobs 4 +# - name: Run chdb/build.sh +# timeout-minutes: 600 +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# source ~/.cargo/env +# pyenv shell 3.8 +# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# bash gen_manifest.sh +# bash ./chdb/build.sh +# pyenv shell 3.8 +# bash -x ./chdb/test_smoke.sh +# - name: Run chdb/build/build_static_lib.sh +# timeout-minutes: 600 +# run: | +# export PATH=$HOME/.pyenv/bin:$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# source ~/.cargo/env +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# bash ./chdb/build/build_static_lib.sh +# pyenv shell --unset +# continue-on-error: false +# - name: Debug libchdb +# run: | +# ls -lh +# llvm-nm libchdb.so | grep query_stable || true +# echo "Global Symbol in libchdb.so:" +# llvm-nm -g libchdb.so || true +# echo "Global Symbol in libclickhouse-local-chdb.a:" +# llvm-nm -g buildlib/programs/local/libclickhouse-local-chdb.a || true +# echo "Global Symbol in libclickhouse-local-lib.a:" +# llvm-nm -g buildlib/programs/local/libclickhouse-local-lib.a || true +# echo "pychdb_cmd.sh:" +# cat buildlib/pychdb_cmd.sh +# echo "libchdb_cmd.sh:" +# cat buildlib/libchdb_cmd.sh +# - name: Scan chdb libraries with grype +# run: | +# echo "Scanning chdb libraries for vulnerabilities..." +# # Files to scan +# FILES_TO_SCAN="" +# [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" +# [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" +# FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)" +# SQLITE_VULNERABILITIES_FOUND=false +# for file in $FILES_TO_SCAN; do +# if [ -f "$file" ]; then +# echo "=== Scanning $file ===" +# SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) +# echo "$SCAN_OUTPUT" +# if echo "$SCAN_OUTPUT" | grep -qi sqlite; then +# echo "❌ SQLite vulnerability found in $file" +# SQLITE_VULNERABILITIES_FOUND=true +# fi +# fi +# done +# if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then +# echo "❌ SQLite vulnerabilities detected in chdb libraries!" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found in chdb libraries" +# fi +# continue-on-error: false +# - name: Run libchdb stub in examples dir +# run: | +# bash -x ./examples/runStub.sh +# - name: Build wheels +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# pyenv shell 3.8 +# make wheel +# - name: Fix wheel platform tag +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# python -m wheel tags --platform-tag=macosx_11_0_arm64 --remove dist/*.whl +# - name: Setup core dump +# run: | +# mkdir -p tmp/core +# sudo sysctl kern.corefile=$PWD/tmp/core/core.%P +# sudo sysctl kern.coredump=1 +# ulimit -c unlimited +# - name: Test wheel on all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# echo "Testing chdb on Python $version" +# pyenv shell $version +# python -m pip install dist/*.whl --force-reinstall +# python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" +# make test +# pyenv shell --unset +# done +# continue-on-error: false +# - name: Check and upload core files if present +# if: always() +# run: | +# if ls tmp/core/core.* >/dev/null 2>&1; then +# echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV +# tar -czvf core-files-macos-arm64.tar.gz tmp/core/core.* +# echo "Core files tar created: core-files-macos-arm64.tar.gz" +# ls -lh core-files-macos-arm64.tar.gz +# else +# echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV +# echo "No core files found in tmp/core" +# fi +# continue-on-error: true +# - name: Upload core files artifact +# if: always() && env.CORE_FILES_FOUND == 'true' +# uses: actions/upload-artifact@v4 +# with: +# name: core-files-macos-arm64 +# path: core-files-macos-arm64.tar.gz +# - name: Show files +# run: ls -lh dist +# shell: bash +# - name: Upload wheels to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} dist/*.whl --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Packege libchdb.so +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf macos-arm64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp +# - name: Package libchdb.a +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf macos-arm64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp +# - name: Upload libchdb.so to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} macos-arm64-libchdb.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Upload libchdb.a to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} macos-arm64-libchdb-static.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - uses: actions/upload-artifact@v4 +# with: +# name: chdb-artifacts-macos-arm64 +# path: | +# ./dist/*.whl +# ./macos-arm64-libchdb.tar.gz +# ./macos-arm64-libchdb-static.tar.gz +# overwrite: true +# - name: Upload pypi +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# python -m twine upload dist/*.whl +# env: +# TWINE_USERNAME: __token__ +# TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/build_macos_x86_wheels.yml b/.github/workflows/build_macos_x86_wheels.yml index 8217b1ad033..e42a743d180 100644 --- a/.github/workflows/build_macos_x86_wheels.yml +++ b/.github/workflows/build_macos_x86_wheels.yml @@ -1,298 +1,298 @@ -name: Build macOS X86 +# name: Build macOS X86 -on: - workflow_dispatch: - inputs: - TAG_NAME: - description: 'Release Version Tag' - required: true - release: - types: [created] - push: - branches: - - main - paths-ignore: - - '**/*.md' - pull_request: - branches: - - main - paths-ignore: - - '**/*.md' +# on: +# workflow_dispatch: +# inputs: +# TAG_NAME: +# description: 'Release Version Tag' +# required: true +# release: +# types: [created] +# push: +# branches: +# - main +# paths-ignore: +# - '**/*.md' +# pull_request: +# branches: +# - main +# paths-ignore: +# - '**/*.md' -jobs: - build_universal_wheel: - name: Build Universal Wheel (macOS x86_64) - runs-on: macos-14-large - steps: - - name: Setup pyenv - run: | - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" +# jobs: +# build_universal_wheel: +# name: Build Universal Wheel (macOS x86_64) +# runs-on: macos-14-large +# steps: +# - name: Setup pyenv +# run: | +# curl https://pyenv.run | bash +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" - pyenv install 3.8:latest - pyenv install 3.9:latest - pyenv install 3.10:latest - pyenv install 3.11:latest - pyenv install 3.12:latest - pyenv install 3.13:latest - pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 +# pyenv install 3.8:latest +# pyenv install 3.9:latest +# pyenv install 3.10:latest +# pyenv install 3.11:latest +# pyenv install 3.12:latest +# pyenv install 3.13:latest +# pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 - echo "Installed versions:" - pyenv versions - - name: Verify pyenv installations - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - echo "Installed Python versions:" - pyenv versions - echo "" - echo "Verifying all required Python versions are available:" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - if ! pyenv versions --bare | grep -q "^$version"; then - echo "ERROR: Python $version is not installed!" - exit 1 - fi - echo "✓ Python $version is installed" - done - echo "All Python versions verified successfully!" - - name: Install dependencies for all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - echo "Installing dependencies for Python $version" - pyenv shell $version - python -m pip install --upgrade pip - python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel>=0.40.0 - pyenv shell --unset - done - - name: Remove /usr/local/bin/python3 - run: | - sudo rm -f /usr/local/bin/python3 - - name: Install clang++ for macOS - run: | - pwd - uname -a - export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 - brew update - brew install ca-certificates lz4 mpdecimal openssl@3 readline sqlite xz z3 zstd - brew install --ignore-dependencies llvm@19 - brew install git ninja libtool gettext gcc binutils grep findutils nasm lld@19 libiconv - brew install ccache || echo "ccache installation failed, continuing without it" - brew install go - cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm - export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:$PATH - which clang++ - clang++ --version - which go - go version - ccache -s || echo "ccache not available yet" - - name: Scan SQLite vulnerabilities with grype - run: | - # Install grype - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin - # Update grype vulnerability database - grype db update - # Check SQLite vulnerabilities in Homebrew packages - echo "Scanning SQLite packages for vulnerabilities..." - GRYPE_RAW_OUTPUT=$(grype dir:/opt/homebrew --scope all-layers 2>/dev/null || true) - echo "Raw grype output:" - echo "$GRYPE_RAW_OUTPUT" - SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) - if [ -n "$SQLITE_SCAN_OUTPUT" ]; then - echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." - echo "SQLite vulnerability details:" - echo "$SQLITE_SCAN_OUTPUT" - exit 1 - else - echo "✅ No SQLite vulnerabilities found" - fi - continue-on-error: false - - name: Upgrade Rust toolchain - run: | - rustup toolchain install nightly-2025-07-07 - rustup default nightly-2025-07-07 - rustup component add rust-src - rustc --version - cargo --version - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Update submodules - run: | - git submodule update --init --recursive --jobs 4 - - name: Run chdb/build.sh - timeout-minutes: 600 - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - source ~/.cargo/env - pyenv shell 3.8 - export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - bash gen_manifest.sh - bash ./chdb/build.sh - pyenv shell 3.8 - bash -x ./chdb/test_smoke.sh - continue-on-error: false - - name: Run chdb/build/build_static_lib.sh - timeout-minutes: 600 - run: | - export PATH=$HOME/.pyenv/bin:$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - source ~/.cargo/env - eval "$(pyenv init -)" - pyenv shell 3.8 - bash ./chdb/build/build_static_lib.sh - pyenv shell --unset - continue-on-error: false - - name: Debug libchdb - run: | - ls -lh - llvm-nm libchdb.so | grep query_stable || true - echo "Global Symbol in libchdb.so:" - llvm-nm -g libchdb.so || true - echo "Global Symbol in libclickhouse-local-chdb.a:" - llvm-nm -g buildlib/programs/local/libclickhouse-local-chdb.a || true - echo "Global Symbol in libclickhouse-local-lib.a:" - llvm-nm -g buildlib/programs/local/libclickhouse-local-lib.a || true - echo "pychdb_cmd.sh:" - cat buildlib/pychdb_cmd.sh - echo "libchdb_cmd.sh:" - cat buildlib/libchdb_cmd.sh - - name: Scan chdb libraries with grype - run: | - echo "Scanning chdb libraries for vulnerabilities..." - # Files to scan - FILES_TO_SCAN="" - [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" - [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" - FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)" - SQLITE_VULNERABILITIES_FOUND=false - for file in $FILES_TO_SCAN; do - if [ -f "$file" ]; then - echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) - echo "$SCAN_OUTPUT" - if echo "$SCAN_OUTPUT" | grep -qi sqlite; then - echo "❌ SQLite vulnerability found in $file" - SQLITE_VULNERABILITIES_FOUND=true - fi - fi - done - if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then - echo "❌ SQLite vulnerabilities detected in chdb libraries!" - exit 1 - else - echo "✅ No SQLite vulnerabilities found in chdb libraries" - fi - continue-on-error: false - - name: Run libchdb stub in examples dir - run: | - bash -x ./examples/runStub.sh - - name: Build wheels - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - source ~/.cargo/env - export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - pyenv shell 3.8 - make wheel - - name: Fix wheel platform tag - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - python -m wheel tags --platform-tag=macosx_10_15_x86_64 --remove dist/*.whl - - name: Setup core dump collection - run: | - mkdir -p tmp/core - sudo sysctl kern.corefile=$PWD/tmp/core/core.%P - sudo sysctl kern.coredump=1 - ulimit -c unlimited - - name: Test wheel on all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - echo "Testing chdb on Python $version" - pyenv shell $version - python -m pip install dist/*.whl --force-reinstall - python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" - make test - pyenv shell --unset - done - continue-on-error: false - - name: Check and upload core files if present - if: always() - run: | - if ls tmp/core/core.* >/dev/null 2>&1; then - echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV - tar -czvf core-files-macos-x86_64.tar.gz tmp/core/core.* - echo "Core files tar created: core-files-macos-x86_64.tar.gz" - ls -lh core-files-macos-x86_64.tar.gz - else - echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV - echo "No core files found in tmp/core" - fi - continue-on-error: true - - name: Upload core files artifact - if: always() && env.CORE_FILES_FOUND == 'true' - uses: actions/upload-artifact@v4 - with: - name: core-files-macos-x86_64 - path: core-files-macos-x86_64.tar.gz - - name: Show files - run: ls -lh dist - shell: bash - - name: Upload wheels to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} dist/*.whl --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Packege libchdb.so - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf macos-x86_64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp - - name: Package libchdb.a - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf macos-x86_64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp - - name: Upload libchdb.so to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} macos-x86_64-libchdb.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Upload libchdb.a to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} macos-x86_64-libchdb-static.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - uses: actions/upload-artifact@v4 - with: - name: chdb-artifacts-macos-x86_64 - path: | - ./dist/*.whl - ./macos-x86_64-libchdb.tar.gz - ./macos-x86_64-libchdb-static.tar.gz - overwrite: true - - name: Upload pypi - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - python -m twine upload dist/*.whl - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} +# echo "Installed versions:" +# pyenv versions +# - name: Verify pyenv installations +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# echo "Installed Python versions:" +# pyenv versions +# echo "" +# echo "Verifying all required Python versions are available:" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# if ! pyenv versions --bare | grep -q "^$version"; then +# echo "ERROR: Python $version is not installed!" +# exit 1 +# fi +# echo "✓ Python $version is installed" +# done +# echo "All Python versions verified successfully!" +# - name: Install dependencies for all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# echo "Installing dependencies for Python $version" +# pyenv shell $version +# python -m pip install --upgrade pip +# python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel>=0.40.0 +# pyenv shell --unset +# done +# - name: Remove /usr/local/bin/python3 +# run: | +# sudo rm -f /usr/local/bin/python3 +# - name: Install clang++ for macOS +# run: | +# pwd +# uname -a +# export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 +# brew update +# brew install ca-certificates lz4 mpdecimal openssl@3 readline sqlite xz z3 zstd +# brew install --ignore-dependencies llvm@19 +# brew install git ninja libtool gettext gcc binutils grep findutils nasm lld@19 libiconv +# brew install ccache || echo "ccache installation failed, continuing without it" +# brew install go +# cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm +# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:$PATH +# which clang++ +# clang++ --version +# which go +# go version +# ccache -s || echo "ccache not available yet" +# - name: Scan SQLite vulnerabilities with grype +# run: | +# # Install grype +# curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin +# # Update grype vulnerability database +# grype db update +# # Check SQLite vulnerabilities in Homebrew packages +# echo "Scanning SQLite packages for vulnerabilities..." +# GRYPE_RAW_OUTPUT=$(grype dir:/opt/homebrew --scope all-layers 2>/dev/null || true) +# echo "Raw grype output:" +# echo "$GRYPE_RAW_OUTPUT" +# SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) +# if [ -n "$SQLITE_SCAN_OUTPUT" ]; then +# echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." +# echo "SQLite vulnerability details:" +# echo "$SQLITE_SCAN_OUTPUT" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found" +# fi +# continue-on-error: false +# - name: Upgrade Rust toolchain +# run: | +# rustup toolchain install nightly-2025-07-07 +# rustup default nightly-2025-07-07 +# rustup component add rust-src +# rustc --version +# cargo --version +# - uses: actions/checkout@v3 +# with: +# fetch-depth: 0 +# - name: Update submodules +# run: | +# git submodule update --init --recursive --jobs 4 +# - name: Run chdb/build.sh +# timeout-minutes: 600 +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# source ~/.cargo/env +# pyenv shell 3.8 +# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# bash gen_manifest.sh +# bash ./chdb/build.sh +# pyenv shell 3.8 +# bash -x ./chdb/test_smoke.sh +# continue-on-error: false +# - name: Run chdb/build/build_static_lib.sh +# timeout-minutes: 600 +# run: | +# export PATH=$HOME/.pyenv/bin:$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# source ~/.cargo/env +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# bash ./chdb/build/build_static_lib.sh +# pyenv shell --unset +# continue-on-error: false +# - name: Debug libchdb +# run: | +# ls -lh +# llvm-nm libchdb.so | grep query_stable || true +# echo "Global Symbol in libchdb.so:" +# llvm-nm -g libchdb.so || true +# echo "Global Symbol in libclickhouse-local-chdb.a:" +# llvm-nm -g buildlib/programs/local/libclickhouse-local-chdb.a || true +# echo "Global Symbol in libclickhouse-local-lib.a:" +# llvm-nm -g buildlib/programs/local/libclickhouse-local-lib.a || true +# echo "pychdb_cmd.sh:" +# cat buildlib/pychdb_cmd.sh +# echo "libchdb_cmd.sh:" +# cat buildlib/libchdb_cmd.sh +# - name: Scan chdb libraries with grype +# run: | +# echo "Scanning chdb libraries for vulnerabilities..." +# # Files to scan +# FILES_TO_SCAN="" +# [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" +# [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" +# FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)" +# SQLITE_VULNERABILITIES_FOUND=false +# for file in $FILES_TO_SCAN; do +# if [ -f "$file" ]; then +# echo "=== Scanning $file ===" +# SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) +# echo "$SCAN_OUTPUT" +# if echo "$SCAN_OUTPUT" | grep -qi sqlite; then +# echo "❌ SQLite vulnerability found in $file" +# SQLITE_VULNERABILITIES_FOUND=true +# fi +# fi +# done +# if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then +# echo "❌ SQLite vulnerabilities detected in chdb libraries!" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found in chdb libraries" +# fi +# continue-on-error: false +# - name: Run libchdb stub in examples dir +# run: | +# bash -x ./examples/runStub.sh +# - name: Build wheels +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# source ~/.cargo/env +# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# pyenv shell 3.8 +# make wheel +# - name: Fix wheel platform tag +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# python -m wheel tags --platform-tag=macosx_10_15_x86_64 --remove dist/*.whl +# - name: Setup core dump collection +# run: | +# mkdir -p tmp/core +# sudo sysctl kern.corefile=$PWD/tmp/core/core.%P +# sudo sysctl kern.coredump=1 +# ulimit -c unlimited +# - name: Test wheel on all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do +# echo "Testing chdb on Python $version" +# pyenv shell $version +# python -m pip install dist/*.whl --force-reinstall +# python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" +# make test +# pyenv shell --unset +# done +# continue-on-error: false +# - name: Check and upload core files if present +# if: always() +# run: | +# if ls tmp/core/core.* >/dev/null 2>&1; then +# echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV +# tar -czvf core-files-macos-x86_64.tar.gz tmp/core/core.* +# echo "Core files tar created: core-files-macos-x86_64.tar.gz" +# ls -lh core-files-macos-x86_64.tar.gz +# else +# echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV +# echo "No core files found in tmp/core" +# fi +# continue-on-error: true +# - name: Upload core files artifact +# if: always() && env.CORE_FILES_FOUND == 'true' +# uses: actions/upload-artifact@v4 +# with: +# name: core-files-macos-x86_64 +# path: core-files-macos-x86_64.tar.gz +# - name: Show files +# run: ls -lh dist +# shell: bash +# - name: Upload wheels to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} dist/*.whl --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Packege libchdb.so +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf macos-x86_64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp +# - name: Package libchdb.a +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf macos-x86_64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp +# - name: Upload libchdb.so to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} macos-x86_64-libchdb.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Upload libchdb.a to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} macos-x86_64-libchdb-static.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - uses: actions/upload-artifact@v4 +# with: +# name: chdb-artifacts-macos-x86_64 +# path: | +# ./dist/*.whl +# ./macos-x86_64-libchdb.tar.gz +# ./macos-x86_64-libchdb-static.tar.gz +# overwrite: true +# - name: Upload pypi +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# python -m twine upload dist/*.whl +# env: +# TWINE_USERNAME: __token__ +# TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/build_musllinux_x86_wheels.yml b/.github/workflows/build_musllinux_x86_wheels.yml new file mode 100644 index 00000000000..b5c7cb656b7 --- /dev/null +++ b/.github/workflows/build_musllinux_x86_wheels.yml @@ -0,0 +1,265 @@ +name: Build Linux(musllinux) x86 + +on: + workflow_dispatch: + inputs: + TAG_NAME: + description: 'Release Version Tag' + required: true + release: + types: [created] + push: + branches: + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - main + paths-ignore: + - '**/*.md' + + +jobs: + build_musllinux_wheels: + name: Build musllinux wheels (Alpine Linux x86_64) + runs-on: gh-64c + container: + image: quay.io/pypa/musllinux_1_2_x86_64 + options: --privileged + steps: + - name: Check system info + run: | + echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" + if [ -f /lib/ld-musl-x86_64.so.1 ]; then + echo "musl libc x86_64" + elif [ -f /lib/libc.musl-x86_64.so.1 ]; then + echo "musl libc x86_64" + else + echo "Not musl libc" + fi + - name: Install Python build dependencies + run: | + apk update + apk add --no-cache make build-base openssl-dev zlib-dev \ + bzip2-dev readline-dev sqlite-dev wget curl llvm \ + ncurses-dev xz-dev tk-dev libxml2-dev \ + libffi-dev linux-headers + - name: Scan SQLite vulnerabilities with grype + run: | + # Install grype and required tools + curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin + + # Update grype vulnerability database + grype db update + + # Check SQLite vulnerabilities in installed packages + echo "Scanning SQLite packages for vulnerabilities..." + GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) + echo "Raw grype output:" + echo "$GRYPE_RAW_OUTPUT" + + SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) + + if [ -n "$SQLITE_SCAN_OUTPUT" ]; then + echo "SQLite vulnerabilities found in packages! Build should be reviewed." + echo "SQLite vulnerability details:" + echo "$SQLITE_SCAN_OUTPUT" + else + echo "No SQLite vulnerabilities found" + fi + continue-on-error: false + - name: Setup pyenv + run: | + curl https://pyenv.run | bash + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv install 3.8:latest + pyenv install 3.9:latest + pyenv install 3.10:latest + pyenv install 3.11:latest + pyenv install 3.12:latest + pyenv install 3.13:latest + pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 + + # Verify installations + echo "Installed versions:" + pyenv versions + - name: Verify pyenv installations + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + echo "Installed Python versions:" + pyenv versions + echo "" + echo "Verifying all required Python versions are available:" + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + if ! pyenv versions --bare | grep -q "^$version"; then + echo "ERROR: Python $version is not installed!" + exit 1 + fi + echo "✓ Python $version is installed" + done + echo "All Python versions verified successfully!" + - name: Install dependencies for all Python versions + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + echo "Installing dependencies for Python $version" + pyenv shell $version + python -m pip install --upgrade pip + if [ "$version" = "3.8" ]; then + python -m pip install setuptools tox twine psutil wheel + else + python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel + fi + pyenv shell --unset + done + - name: Install clang++ for Alpine + run: | + apk add --no-cache make cmake ccache ninja yasm gawk wget + apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 + # Install Rust toolchain via rustup for proper target management + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + source $HOME/.cargo/env + # Add the correct musl target for Alpine + rustup target add x86_64-unknown-linux-musl + rustc --version + cargo --version + ccache -s + - name: Update git + run: | + apk add --no-cache git + git --version + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Configure git safe directory + run: | + git config --global --add safe.directory '*' + - name: Update submodules + run: | + git submodule update --init --recursive --jobs 4 + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: musllinux-1-2-x86_64 + max-size: 5G + append-timestamp: true + - name: setup clang and link clang-20 to clang + run: | + ln -sf /usr/bin/clang-20 /usr/bin/clang + ln -sf /usr/bin/clang++-20 /usr/bin/clang++ + which clang++ + clang++ --version + - name: Run chdb/build-musl.sh + timeout-minutes: 600 + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + # Setup Rust environment + source $HOME/.cargo/env + pyenv shell 3.8 + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + bash ./chdb/build-musl.sh + pyenv shell 3.8 + bash -x ./chdb/test_smoke.sh + continue-on-error: false + - name: Scan chdb libraries with grype + run: | + echo "Scanning chdb libraries for vulnerabilities..." + + # Files to scan + FILES_TO_SCAN="" + [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" + [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" + FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" + + SQLITE_VULNERABILITIES_FOUND=false + + for file in $FILES_TO_SCAN; do + if [ -f "$file" ]; then + echo "=== Scanning $file ===" + SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) + echo "$SCAN_OUTPUT" + + if echo "$SCAN_OUTPUT" | grep -qi sqlite; then + echo "SQLite vulnerability found in $file" + SQLITE_VULNERABILITIES_FOUND=true + fi + fi + done + + if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then + echo "SQLite vulnerabilities detected in chdb libraries!" + else + echo "No SQLite vulnerabilities found in chdb libraries" + fi + continue-on-error: false + - name: Check ccache statistics + run: | + ccache -s + ls -lh chdb + df -h + - name: Build wheels + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + pyenv shell 3.8 + make wheel + - name: Install patchelf from github + run: | + wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz -O patchelf.tar.gz + tar -xvf patchelf.tar.gz + cp bin/patchelf /usr/bin/ + chmod +x /usr/bin/patchelf + patchelf --version + - name: Audit wheels + run: | + python3 -m pip install auditwheel + auditwheel -v repair -w dist/ --plat musllinux_1_2_x86_64 dist/*.whl + continue-on-error: false + - name: Show files + run: | + rm -f dist/*-linux_x86_64.whl + ls -lh dist + shell: bash + - name: Test wheel on all Python versions + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + for version in 3.9 3.10 3.11 3.12 3.13; do + echo "Testing chdb on Python $version" + pyenv shell $version + python -m pip install dist/*.whl --force-reinstall + python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" + make test + pyenv shell --unset + done + continue-on-error: false + - name: Upload wheels to release + if: startsWith(github.ref, 'refs/tags/v') + run: | + gh release upload ${{ github.ref_name }} dist/*.whl --clobber + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + - uses: actions/upload-artifact@v4 + with: + name: chdb-artifacts-musllinux-x86_64 + path: | + ./dist/*.whl + overwrite: true + - name: Upload pypi + if: startsWith(github.ref, 'refs/tags/v') + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv shell 3.8 + python -m twine upload dist/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} \ No newline at end of file diff --git a/chdb/vars.sh b/chdb/vars.sh index 19392da05bf..b1b2100a5b6 100755 --- a/chdb/vars.sh +++ b/chdb/vars.sh @@ -6,8 +6,7 @@ CHDB_DIR="$PROJ_DIR/chdb" # chdb directory CHDB_PY_MOD="_chdb" CHDB_PY_MODULE="${CHDB_PY_MOD}.abi3.so" pushd ${PROJ_DIR} -# CHDB_VERSION=$(python3 -c 'import setup; print(setup.get_latest_git_tag())') -CHDB_VERSION='3.6.0' +CHDB_VERSION=$(python3 -c 'import setup; print(setup.get_latest_git_tag())') popd # try to use largest llvm-strip version diff --git a/setup.py b/setup.py index 84b2836a6b7..9b3d601f389 100644 --- a/setup.py +++ b/setup.py @@ -145,8 +145,7 @@ def build_extensions(self): ), ] # fix the version in chdb/__init__.py - # versionStr = get_latest_git_tag() - versionStr = "3.6.0" + versionStr = get_latest_git_tag() # Call the function to update pyproject.toml # update_pyproject_version(versionStr) From 4fa47341e0b2320d7a3f2ab068b5ebab6e39c27b Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Mon, 13 Oct 2025 22:39:15 +0800 Subject: [PATCH 06/41] chore: update rustup --- .github/workflows/build_musllinux_x86_wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_musllinux_x86_wheels.yml b/.github/workflows/build_musllinux_x86_wheels.yml index b5c7cb656b7..2cce5edc8a3 100644 --- a/.github/workflows/build_musllinux_x86_wheels.yml +++ b/.github/workflows/build_musllinux_x86_wheels.yml @@ -123,8 +123,8 @@ jobs: # Install Rust toolchain via rustup for proper target management curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable source $HOME/.cargo/env - # Add the correct musl target for Alpine - rustup target add x86_64-unknown-linux-musl + rustup toolchain install nightly-2025-07-07 + rustup component add --toolchain nightly-2025-07-07 rust-src rustc --version cargo --version ccache -s From 1649a370ca31f9acfa6cf1b9950498ffef5b2990 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Mon, 13 Oct 2025 23:55:26 +0800 Subject: [PATCH 07/41] chore: update workflow --- .github/workflows/build_musllinux_x86_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_musllinux_x86_wheels.yml b/.github/workflows/build_musllinux_x86_wheels.yml index 2cce5edc8a3..0f7d5fd2791 100644 --- a/.github/workflows/build_musllinux_x86_wheels.yml +++ b/.github/workflows/build_musllinux_x86_wheels.yml @@ -164,7 +164,7 @@ jobs: export CC=/usr/bin/clang export CXX=/usr/bin/clang++ bash ./chdb/build-musl.sh - pyenv shell 3.8 + pyenv shell 3.9 bash -x ./chdb/test_smoke.sh continue-on-error: false - name: Scan chdb libraries with grype From a4576e83a91a8cbc00cb59f6c8702a7e0dc38366 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Tue, 14 Oct 2025 00:44:51 +0800 Subject: [PATCH 08/41] chore: update workflow --- .../build_musllinux_arm64_wheels.yml | 264 ++++++++++++++++++ .../workflows/build_musllinux_x86_wheels.yml | 14 +- 2 files changed, 270 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/build_musllinux_arm64_wheels.yml diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml new file mode 100644 index 00000000000..fd845426995 --- /dev/null +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -0,0 +1,264 @@ +name: Build Linux(musllinux) ARM64 + +on: + workflow_dispatch: + inputs: + TAG_NAME: + description: 'Release Version Tag' + required: true + release: + types: [created] + push: + branches: + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - main + paths-ignore: + - '**/*.md' + + +jobs: + build_musllinux_wheels: + name: Build musllinux wheels (Alpine Linux aarch64) + runs-on: GH-Linux-ARM64 + container: + image: quay.io/pypa/musllinux_1_2_aarch64 + options: --privileged + steps: + - name: Check system info + run: | + echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" + if [ -f /lib/ld-musl-aarch64.so.1 ]; then + echo "musl libc aarch64" + elif [ -f /lib/libc.musl-aarch64.so.1 ]; then + echo "musl libc aarch64" + else + echo "Not musl libc" + fi + - name: Install Python build dependencies + run: | + apk update + apk add --no-cache make build-base openssl-dev zlib-dev \ + bzip2-dev readline-dev sqlite-dev wget curl llvm \ + ncurses-dev xz-dev tk-dev libxml2-dev \ + libffi-dev linux-headers + - name: Scan SQLite vulnerabilities with grype + run: | + # Install grype and required tools + curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin + + # Update grype vulnerability database + grype db update + + # Check SQLite vulnerabilities in installed packages + echo "Scanning SQLite packages for vulnerabilities..." + GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) + echo "Raw grype output:" + echo "$GRYPE_RAW_OUTPUT" + + SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) + + if [ -n "$SQLITE_SCAN_OUTPUT" ]; then + echo "SQLite vulnerabilities found in packages! Build should be reviewed." + echo "SQLite vulnerability details:" + echo "$SQLITE_SCAN_OUTPUT" + else + echo "No SQLite vulnerabilities found" + fi + continue-on-error: false + - name: Setup pyenv + run: | + curl https://pyenv.run | bash + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv install 3.8:latest + pyenv install 3.9:latest + pyenv install 3.10:latest + pyenv install 3.11:latest + pyenv install 3.12:latest + pyenv install 3.13:latest + pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 + + # Verify installations + echo "Installed versions:" + pyenv versions + - name: Verify pyenv installations + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + echo "Installed Python versions:" + pyenv versions + echo "" + echo "Verifying all required Python versions are available:" + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + if ! pyenv versions --bare | grep -q "^$version"; then + echo "ERROR: Python $version is not installed!" + exit 1 + fi + echo "✓ Python $version is installed" + done + echo "All Python versions verified successfully!" + - name: Install dependencies for all Python versions + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + echo "Installing dependencies for Python $version" + pyenv shell $version + python -m pip install --upgrade pip + if [ "$version" = "3.8" ]; then + python -m pip install setuptools tox twine psutil wheel + else + python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel + fi + pyenv shell --unset + done + - name: Install clang++ for Alpine + run: | + apk add --no-cache make cmake ccache ninja yasm gawk wget + apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 + # Install Rust toolchain via rustup for proper target management + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + source $HOME/.cargo/env + rustup toolchain install nightly-2025-07-07 + rustup component add --toolchain nightly-2025-07-07 rust-src + rustc --version + cargo --version + ccache -s + - name: Update git + run: | + apk add --no-cache git + git --version + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Configure git safe directory + run: | + git config --global --add safe.directory '*' + - name: Update submodules + run: | + git submodule update --init --recursive --jobs 4 + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: musllinux-1-2-aarch64 + max-size: 5G + append-timestamp: true + - name: setup clang and link clang-20 to clang + run: | + ln -sf /usr/bin/clang-20 /usr/bin/clang + ln -sf /usr/bin/clang++-20 /usr/bin/clang++ + which clang++ + clang++ --version + - name: Run chdb/build-musl.sh + timeout-minutes: 600 + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + # Setup Rust environment + source $HOME/.cargo/env + pyenv shell 3.8 + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + bash ./chdb/build-musl.sh + pyenv shell 3.9 + bash -x ./chdb/test_smoke.sh + continue-on-error: false + - name: Scan chdb libraries with grype + run: | + echo "Scanning chdb libraries for vulnerabilities..." + + FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" + + SQLITE_VULNERABILITIES_FOUND=false + + for file in $FILES_TO_SCAN; do + if [ -f "$file" ]; then + echo "=== Scanning $file ===" + SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) + echo "$SCAN_OUTPUT" + + if echo "$SCAN_OUTPUT" | grep -qi sqlite; then + echo "SQLite vulnerability found in $file" + SQLITE_VULNERABILITIES_FOUND=true + fi + fi + done + + if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then + echo "SQLite vulnerabilities detected in chdb libraries!" + else + echo "No SQLite vulnerabilities found in chdb libraries" + fi + continue-on-error: false + - name: Check ccache statistics + run: | + ccache -s + ls -lh chdb + df -h + - name: Build wheels + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + pyenv shell 3.8 + make wheel + - name: Install patchelf from github + run: | + wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz + tar -xvf patchelf.tar.gz + cp bin/patchelf /usr/bin/ + chmod +x /usr/bin/patchelf + patchelf --version + - name: Audit wheels + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv shell 3.13 + python -m pip install auditwheel + auditwheel -v repair -w dist/ --plat musllinux_1_2_aarch64 dist/*.whl + continue-on-error: false + - name: Show files + run: | + rm -f dist/*-linux_aarch64.whl + ls -lh dist + shell: bash + - name: Test wheel on all Python versions + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + for version in 3.9 3.10 3.11 3.12 3.13; do + echo "Testing chdb on Python $version" + pyenv shell $version + python -m pip install dist/*.whl --force-reinstall + python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" + make test + pyenv shell --unset + done + continue-on-error: false + - name: Upload wheels to release + if: startsWith(github.ref, 'refs/tags/v') + run: | + gh release upload ${{ github.ref_name }} dist/*.whl --clobber + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + - uses: actions/upload-artifact@v4 + with: + name: chdb-artifacts-musllinux-aarch64 + path: | + ./dist/*.whl + overwrite: true + - name: Upload pypi + if: startsWith(github.ref, 'refs/tags/v') + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv shell 3.13 + python -m twine upload dist/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/build_musllinux_x86_wheels.yml b/.github/workflows/build_musllinux_x86_wheels.yml index 0f7d5fd2791..bfaabea2f8f 100644 --- a/.github/workflows/build_musllinux_x86_wheels.yml +++ b/.github/workflows/build_musllinux_x86_wheels.yml @@ -158,7 +158,6 @@ jobs: run: | export PATH="$HOME/.pyenv/bin:$PATH" eval "$(pyenv init -)" - # Setup Rust environment source $HOME/.cargo/env pyenv shell 3.8 export CC=/usr/bin/clang @@ -171,10 +170,6 @@ jobs: run: | echo "Scanning chdb libraries for vulnerabilities..." - # Files to scan - FILES_TO_SCAN="" - [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" - [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" SQLITE_VULNERABILITIES_FOUND=false @@ -220,7 +215,10 @@ jobs: patchelf --version - name: Audit wheels run: | - python3 -m pip install auditwheel + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv shell 3.13 + python -m pip install auditwheel auditwheel -v repair -w dist/ --plat musllinux_1_2_x86_64 dist/*.whl continue-on-error: false - name: Show files @@ -258,8 +256,8 @@ jobs: run: | export PATH="$HOME/.pyenv/bin:$PATH" eval "$(pyenv init -)" - pyenv shell 3.8 + pyenv shell 3.13 python -m twine upload dist/*.whl env: TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} \ No newline at end of file + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} From 76ff55162ec86c21ee32d4dc3da083ed15226440 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Tue, 14 Oct 2025 00:58:03 +0800 Subject: [PATCH 09/41] chore: update workflow --- .github/workflows/build_musllinux_arm64_wheels.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index fd845426995..0192f874ee8 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -103,6 +103,12 @@ jobs: echo "All Python versions verified successfully!" - name: Install dependencies for all Python versions run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + source $HOME/.cargo/env + rustup toolchain install nightly-2025-07-07 + rustup component add --toolchain nightly-2025-07-07 rust-src + rustc --version + cargo --version export PATH="$HOME/.pyenv/bin:$PATH" eval "$(pyenv init -)" for version in 3.8 3.9 3.10 3.11 3.12 3.13; do @@ -120,13 +126,6 @@ jobs: run: | apk add --no-cache make cmake ccache ninja yasm gawk wget apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 - # Install Rust toolchain via rustup for proper target management - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable - source $HOME/.cargo/env - rustup toolchain install nightly-2025-07-07 - rustup component add --toolchain nightly-2025-07-07 rust-src - rustc --version - cargo --version ccache -s - name: Update git run: | From f5735805f2f53e204b22460b6490c36e39a3df76 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Tue, 14 Oct 2025 01:38:51 +0800 Subject: [PATCH 10/41] test: fix tests --- tests/test_open_session_after_failure.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/test_open_session_after_failure.py b/tests/test_open_session_after_failure.py index 4935585b668..161b8e9dc49 100644 --- a/tests/test_open_session_after_failure.py +++ b/tests/test_open_session_after_failure.py @@ -2,6 +2,7 @@ import unittest import shutil +import platform from chdb import session @@ -9,6 +10,13 @@ test_dir2 = "/usr/bin" +def is_musl_env(): + if 'musl' in platform.platform().lower(): + return True + + return False + + class TestStateful(unittest.TestCase): def setUp(self) -> None: shutil.rmtree(test_dir1, ignore_errors=True) @@ -20,8 +28,9 @@ def tearDown(self) -> None: def test_path(self): # Test that creating session with invalid path (read-only directory) raises exception - with self.assertRaises(Exception): - sess = session.Session(test_dir2) + if not is_musl_env(): + with self.assertRaises(Exception): + sess = session.Session(test_dir2) # Test that creating session with valid path works after failure sess = session.Session(test_dir1) From 4cd2a5b07b840aebc432425b2d2b046c91490aa0 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Tue, 14 Oct 2025 02:18:58 +0800 Subject: [PATCH 11/41] test: fix tests --- tests/test_open_session_after_failure.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tests/test_open_session_after_failure.py b/tests/test_open_session_after_failure.py index 161b8e9dc49..2c26bd0eed8 100644 --- a/tests/test_open_session_after_failure.py +++ b/tests/test_open_session_after_failure.py @@ -2,7 +2,7 @@ import unittest import shutil -import platform +import os from chdb import session @@ -10,13 +10,6 @@ test_dir2 = "/usr/bin" -def is_musl_env(): - if 'musl' in platform.platform().lower(): - return True - - return False - - class TestStateful(unittest.TestCase): def setUp(self) -> None: shutil.rmtree(test_dir1, ignore_errors=True) @@ -28,7 +21,7 @@ def tearDown(self) -> None: def test_path(self): # Test that creating session with invalid path (read-only directory) raises exception - if not is_musl_env(): + if not os.access(test_dir2, os.W_OK): with self.assertRaises(Exception): sess = session.Session(test_dir2) From 4ad4fe09b60f190da6370000f56f7228c3ed132b Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Tue, 14 Oct 2025 10:46:30 +0800 Subject: [PATCH 12/41] test: fix tests --- .../build_musllinux_arm64_wheels.yml | 198 +++++++++--------- 1 file changed, 99 insertions(+), 99 deletions(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index 0192f874ee8..1dfe05d6053 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -28,110 +28,110 @@ jobs: image: quay.io/pypa/musllinux_1_2_aarch64 options: --privileged steps: - - name: Check system info - run: | - echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" - if [ -f /lib/ld-musl-aarch64.so.1 ]; then - echo "musl libc aarch64" - elif [ -f /lib/libc.musl-aarch64.so.1 ]; then - echo "musl libc aarch64" - else - echo "Not musl libc" - fi - - name: Install Python build dependencies - run: | - apk update - apk add --no-cache make build-base openssl-dev zlib-dev \ - bzip2-dev readline-dev sqlite-dev wget curl llvm \ - ncurses-dev xz-dev tk-dev libxml2-dev \ - libffi-dev linux-headers - - name: Scan SQLite vulnerabilities with grype - run: | - # Install grype and required tools - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin + # - name: Check system info + # run: | + # echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" + # if [ -f /lib/ld-musl-aarch64.so.1 ]; then + # echo "musl libc aarch64" + # elif [ -f /lib/libc.musl-aarch64.so.1 ]; then + # echo "musl libc aarch64" + # else + # echo "Not musl libc" + # fi + # - name: Install Python build dependencies + # run: | + # apk update + # apk add --no-cache make build-base openssl-dev zlib-dev \ + # bzip2-dev readline-dev sqlite-dev wget curl llvm \ + # ncurses-dev xz-dev tk-dev libxml2-dev \ + # libffi-dev linux-headers + # - name: Scan SQLite vulnerabilities with grype + # run: | + # # Install grype and required tools + # curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin - # Update grype vulnerability database - grype db update + # # Update grype vulnerability database + # grype db update - # Check SQLite vulnerabilities in installed packages - echo "Scanning SQLite packages for vulnerabilities..." - GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) - echo "Raw grype output:" - echo "$GRYPE_RAW_OUTPUT" + # # Check SQLite vulnerabilities in installed packages + # echo "Scanning SQLite packages for vulnerabilities..." + # GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) + # echo "Raw grype output:" + # echo "$GRYPE_RAW_OUTPUT" - SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) + # SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) - if [ -n "$SQLITE_SCAN_OUTPUT" ]; then - echo "SQLite vulnerabilities found in packages! Build should be reviewed." - echo "SQLite vulnerability details:" - echo "$SQLITE_SCAN_OUTPUT" - else - echo "No SQLite vulnerabilities found" - fi - continue-on-error: false - - name: Setup pyenv - run: | - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv install 3.8:latest - pyenv install 3.9:latest - pyenv install 3.10:latest - pyenv install 3.11:latest - pyenv install 3.12:latest - pyenv install 3.13:latest - pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 + # if [ -n "$SQLITE_SCAN_OUTPUT" ]; then + # echo "SQLite vulnerabilities found in packages! Build should be reviewed." + # echo "SQLite vulnerability details:" + # echo "$SQLITE_SCAN_OUTPUT" + # else + # echo "No SQLite vulnerabilities found" + # fi + # continue-on-error: false + # - name: Setup pyenv + # run: | + # curl https://pyenv.run | bash + # export PATH="$HOME/.pyenv/bin:$PATH" + # eval "$(pyenv init -)" + # pyenv install 3.8:latest + # pyenv install 3.9:latest + # pyenv install 3.10:latest + # pyenv install 3.11:latest + # pyenv install 3.12:latest + # pyenv install 3.13:latest + # pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 - # Verify installations - echo "Installed versions:" - pyenv versions - - name: Verify pyenv installations - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - echo "Installed Python versions:" - pyenv versions - echo "" - echo "Verifying all required Python versions are available:" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - if ! pyenv versions --bare | grep -q "^$version"; then - echo "ERROR: Python $version is not installed!" - exit 1 - fi - echo "✓ Python $version is installed" - done - echo "All Python versions verified successfully!" - - name: Install dependencies for all Python versions - run: | - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable - source $HOME/.cargo/env - rustup toolchain install nightly-2025-07-07 - rustup component add --toolchain nightly-2025-07-07 rust-src - rustc --version - cargo --version - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - echo "Installing dependencies for Python $version" - pyenv shell $version - python -m pip install --upgrade pip - if [ "$version" = "3.8" ]; then - python -m pip install setuptools tox twine psutil wheel - else - python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel - fi - pyenv shell --unset - done - - name: Install clang++ for Alpine - run: | - apk add --no-cache make cmake ccache ninja yasm gawk wget - apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 - ccache -s - - name: Update git - run: | - apk add --no-cache git - git --version - - uses: actions/checkout@v3 + # # Verify installations + # echo "Installed versions:" + # pyenv versions + # - name: Verify pyenv installations + # run: | + # export PATH="$HOME/.pyenv/bin:$PATH" + # eval "$(pyenv init -)" + # echo "Installed Python versions:" + # pyenv versions + # echo "" + # echo "Verifying all required Python versions are available:" + # for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + # if ! pyenv versions --bare | grep -q "^$version"; then + # echo "ERROR: Python $version is not installed!" + # exit 1 + # fi + # echo "✓ Python $version is installed" + # done + # echo "All Python versions verified successfully!" + # - name: Install dependencies for all Python versions + # run: | + # curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + # source $HOME/.cargo/env + # rustup toolchain install nightly-2025-07-07 + # rustup component add --toolchain nightly-2025-07-07 rust-src + # rustc --version + # cargo --version + # export PATH="$HOME/.pyenv/bin:$PATH" + # eval "$(pyenv init -)" + # for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + # echo "Installing dependencies for Python $version" + # pyenv shell $version + # python -m pip install --upgrade pip + # if [ "$version" = "3.8" ]; then + # python -m pip install setuptools tox twine psutil wheel + # else + # python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel + # fi + # pyenv shell --unset + # done + # - name: Install clang++ for Alpine + # run: | + # apk add --no-cache make cmake ccache ninja yasm gawk wget + # apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 + # ccache -s + # - name: Update git + # run: | + # apk add --no-cache git + # git --version + - uses: taiki-e/checkout-action@v1 with: fetch-depth: 0 - name: Configure git safe directory From 66372853ba10012eba0261f9d7e4fe422f5f03ac Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Tue, 14 Oct 2025 11:24:48 +0800 Subject: [PATCH 13/41] test: fix tests --- .../build_musllinux_arm64_wheels.yml | 303 +++++++++++------- 1 file changed, 184 insertions(+), 119 deletions(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index 1dfe05d6053..093ad9323f0 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -21,12 +21,31 @@ on: jobs: + prepare_host: + name: Prepare host environment + runs-on: GH-Linux-ARM64 + outputs: + artifacts_dir: ${{ steps.create_dir.outputs.artifacts_dir }} + steps: + - name: Create artifacts directory + id: create_dir + run: | + ARTIFACTS_DIR="/tmp/chdb-artifacts-${{ github.run_id }}" + sudo mkdir -p "$ARTIFACTS_DIR" + sudo chmod 777 "$ARTIFACTS_DIR" + echo "Created unique artifacts directory: $ARTIFACTS_DIR" + echo "artifacts_dir=$ARTIFACTS_DIR" >> $GITHUB_OUTPUT + ls -la "$ARTIFACTS_DIR" + build_musllinux_wheels: name: Build musllinux wheels (Alpine Linux aarch64) runs-on: GH-Linux-ARM64 + needs: prepare_host container: image: quay.io/pypa/musllinux_1_2_aarch64 options: --privileged + volumes: + - ${{ needs.prepare_host.outputs.artifacts_dir }}:/host-artifacts steps: # - name: Check system info # run: | @@ -131,133 +150,179 @@ jobs: # run: | # apk add --no-cache git # git --version - - uses: taiki-e/checkout-action@v1 - with: - fetch-depth: 0 - - name: Configure git safe directory - run: | - git config --global --add safe.directory '*' - - name: Update submodules - run: | - git submodule update --init --recursive --jobs 4 - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: musllinux-1-2-aarch64 - max-size: 5G - append-timestamp: true - - name: setup clang and link clang-20 to clang - run: | - ln -sf /usr/bin/clang-20 /usr/bin/clang - ln -sf /usr/bin/clang++-20 /usr/bin/clang++ - which clang++ - clang++ --version - - name: Run chdb/build-musl.sh - timeout-minutes: 600 - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - # Setup Rust environment - source $HOME/.cargo/env - pyenv shell 3.8 - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - bash ./chdb/build-musl.sh - pyenv shell 3.9 - bash -x ./chdb/test_smoke.sh - continue-on-error: false - - name: Scan chdb libraries with grype - run: | - echo "Scanning chdb libraries for vulnerabilities..." + # - uses: taiki-e/checkout-action@v1 + # with: + # fetch-depth: 0 + # - name: Configure git safe directory + # run: | + # git config --global --add safe.directory '*' + # - name: Update submodules + # run: | + # git submodule update --init --recursive --jobs 4 + # # - name: ccache + # # uses: hendrikmuhs/ccache-action@v1.2 + # # with: + # # key: musllinux-1-2-aarch64 + # # max-size: 5G + # # append-timestamp: true + # - name: setup clang and link clang-20 to clang + # run: | + # ln -sf /usr/bin/clang-20 /usr/bin/clang + # ln -sf /usr/bin/clang++-20 /usr/bin/clang++ + # which clang++ + # clang++ --version + # - name: Run chdb/build-musl.sh + # timeout-minutes: 600 + # run: | + # export PATH="$HOME/.pyenv/bin:$PATH" + # eval "$(pyenv init -)" + # # Setup Rust environment + # source $HOME/.cargo/env + # pyenv shell 3.8 + # export CC=/usr/bin/clang + # export CXX=/usr/bin/clang++ + # bash ./chdb/build-musl.sh + # pyenv shell 3.9 + # bash -x ./chdb/test_smoke.sh + # continue-on-error: false + # - name: Scan chdb libraries with grype + # run: | + # echo "Scanning chdb libraries for vulnerabilities..." - FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" + # FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" - SQLITE_VULNERABILITIES_FOUND=false + # SQLITE_VULNERABILITIES_FOUND=false - for file in $FILES_TO_SCAN; do - if [ -f "$file" ]; then - echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) - echo "$SCAN_OUTPUT" + # for file in $FILES_TO_SCAN; do + # if [ -f "$file" ]; then + # echo "=== Scanning $file ===" + # SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) + # echo "$SCAN_OUTPUT" - if echo "$SCAN_OUTPUT" | grep -qi sqlite; then - echo "SQLite vulnerability found in $file" - SQLITE_VULNERABILITIES_FOUND=true - fi - fi - done + # if echo "$SCAN_OUTPUT" | grep -qi sqlite; then + # echo "SQLite vulnerability found in $file" + # SQLITE_VULNERABILITIES_FOUND=true + # fi + # fi + # done - if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then - echo "SQLite vulnerabilities detected in chdb libraries!" - else - echo "No SQLite vulnerabilities found in chdb libraries" - fi - continue-on-error: false - - name: Check ccache statistics - run: | - ccache -s - ls -lh chdb - df -h - - name: Build wheels - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - pyenv shell 3.8 - make wheel - - name: Install patchelf from github - run: | - wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz - tar -xvf patchelf.tar.gz - cp bin/patchelf /usr/bin/ - chmod +x /usr/bin/patchelf - patchelf --version - - name: Audit wheels - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.13 - python -m pip install auditwheel - auditwheel -v repair -w dist/ --plat musllinux_1_2_aarch64 dist/*.whl - continue-on-error: false - - name: Show files - run: | - rm -f dist/*-linux_aarch64.whl - ls -lh dist - shell: bash - - name: Test wheel on all Python versions + # if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then + # echo "SQLite vulnerabilities detected in chdb libraries!" + # else + # echo "No SQLite vulnerabilities found in chdb libraries" + # fi + # continue-on-error: false + # - name: Check ccache statistics + # run: | + # ccache -s + # ls -lh chdb + # df -h + # - name: Build wheels + # run: | + # export PATH="$HOME/.pyenv/bin:$PATH" + # eval "$(pyenv init -)" + # export CC=/usr/bin/clang + # export CXX=/usr/bin/clang++ + # pyenv shell 3.8 + # make wheel + # - name: Install patchelf from github + # run: | + # wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz + # tar -xvf patchelf.tar.gz + # cp bin/patchelf /usr/bin/ + # chmod +x /usr/bin/patchelf + # patchelf --version + # - name: Audit wheels + # run: | + # export PATH="$HOME/.pyenv/bin:$PATH" + # eval "$(pyenv init -)" + # pyenv shell 3.13 + # python -m pip install auditwheel + # auditwheel -v repair -w dist/ --plat musllinux_1_2_aarch64 dist/*.whl + # continue-on-error: false + # - name: Show files + # run: | + # rm -f dist/*-linux_aarch64.whl + # ls -lh dist + # shell: bash + # - name: Test wheel on all Python versions + # run: | + # export PATH="$HOME/.pyenv/bin:$PATH" + # eval "$(pyenv init -)" + # for version in 3.9 3.10 3.11 3.12 3.13; do + # echo "Testing chdb on Python $version" + # pyenv shell $version + # python -m pip install dist/*.whl --force-reinstall + # python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" + # make test + # pyenv shell --unset + # done + # continue-on-error: false + # - name: Upload wheels to release + # if: startsWith(github.ref, 'refs/tags/v') + # run: | + # gh release upload ${{ github.ref_name }} dist/*.whl --clobber + # env: + # GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + - name: Create test file and copy artifacts run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.9 3.10 3.11 3.12 3.13; do - echo "Testing chdb on Python $version" - pyenv shell $version - python -m pip install dist/*.whl --force-reinstall - python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" - make test - pyenv shell --unset - done - continue-on-error: false - - name: Upload wheels to release - if: startsWith(github.ref, 'refs/tags/v') + # mkdir -p /host-artifacts + # cp ./dist/*.whl /host-artifacts/ + # ls -la /host-artifacts/ + echo "Test file created and artifacts copied" > /host-artifacts/test.whl + # - name: Upload pypi + # if: startsWith(github.ref, 'refs/tags/v') + # run: | + # export PATH="$HOME/.pyenv/bin:$PATH" + # eval "$(pyenv init -)" + # pyenv shell 3.13 + # python -m twine upload dist/*.whl + # env: + # TWINE_USERNAME: __token__ + # TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + + upload_artifacts: + name: Upload build artifacts + runs-on: GH-Linux-ARM64 + needs: [prepare_host, build_musllinux_wheels] + steps: + - name: Check artifacts directory run: | - gh release upload ${{ github.ref_name }} dist/*.whl --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - uses: actions/upload-artifact@v4 + ARTIFACTS_DIR="${{ needs.prepare_host.outputs.artifacts_dir }}" + echo "Checking artifacts in directory: $ARTIFACTS_DIR" + ls -la "$ARTIFACTS_DIR" || echo "No artifacts directory found" + - name: Upload artifacts + uses: actions/upload-artifact@v4 with: - name: chdb-artifacts-musllinux-aarch64 + name: chdb-artifacts-musllinux-aarch64-${{ github.run_id }} path: | - ./dist/*.whl + ${{ needs.prepare_host.outputs.artifacts_dir }}/*.whl overwrite: true - - name: Upload pypi - if: startsWith(github.ref, 'refs/tags/v') + + cleanup: + name: Cleanup temporary directories + runs-on: GH-Linux-ARM64 + needs: [prepare_host, build_musllinux_wheels, upload_artifacts] + if: always() + steps: + - name: Remove temporary artifacts directory run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.13 - python -m twine upload dist/*.whl - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + ARTIFACTS_DIR="${{ needs.prepare_host.outputs.artifacts_dir }}" + echo "Cleaning up temporary directory: $ARTIFACTS_DIR" + + if [ -d "$ARTIFACTS_DIR" ]; then + echo "Directory contents before cleanup:" + ls -la "$ARTIFACTS_DIR" + + sudo rm -rf "$ARTIFACTS_DIR" + echo "Successfully removed $ARTIFACTS_DIR" + else + echo "Directory $ARTIFACTS_DIR not found, nothing to clean" + fi + + if [ ! -d "$ARTIFACTS_DIR" ]; then + echo "✅ Cleanup successful - directory removed" + else + echo "❌ Cleanup failed - directory still exists" + exit 1 + fi From dfb89bcab09d3511f251f3178011ce2c91ae915a Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Tue, 14 Oct 2025 12:47:09 +0800 Subject: [PATCH 14/41] test: fix tests --- .../build_musllinux_arm64_wheels.yml | 495 ++++++++---------- 1 file changed, 207 insertions(+), 288 deletions(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index 093ad9323f0..cdd9c687f23 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -19,310 +19,229 @@ on: paths-ignore: - '**/*.md' - jobs: - prepare_host: - name: Prepare host environment - runs-on: GH-Linux-ARM64 - outputs: - artifacts_dir: ${{ steps.create_dir.outputs.artifacts_dir }} - steps: - - name: Create artifacts directory - id: create_dir - run: | - ARTIFACTS_DIR="/tmp/chdb-artifacts-${{ github.run_id }}" - sudo mkdir -p "$ARTIFACTS_DIR" - sudo chmod 777 "$ARTIFACTS_DIR" - echo "Created unique artifacts directory: $ARTIFACTS_DIR" - echo "artifacts_dir=$ARTIFACTS_DIR" >> $GITHUB_OUTPUT - ls -la "$ARTIFACTS_DIR" - build_musllinux_wheels: name: Build musllinux wheels (Alpine Linux aarch64) runs-on: GH-Linux-ARM64 - needs: prepare_host - container: - image: quay.io/pypa/musllinux_1_2_aarch64 - options: --privileged - volumes: - - ${{ needs.prepare_host.outputs.artifacts_dir }}:/host-artifacts steps: - # - name: Check system info - # run: | - # echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" - # if [ -f /lib/ld-musl-aarch64.so.1 ]; then - # echo "musl libc aarch64" - # elif [ -f /lib/libc.musl-aarch64.so.1 ]; then - # echo "musl libc aarch64" - # else - # echo "Not musl libc" - # fi - # - name: Install Python build dependencies - # run: | - # apk update - # apk add --no-cache make build-base openssl-dev zlib-dev \ - # bzip2-dev readline-dev sqlite-dev wget curl llvm \ - # ncurses-dev xz-dev tk-dev libxml2-dev \ - # libffi-dev linux-headers - # - name: Scan SQLite vulnerabilities with grype - # run: | - # # Install grype and required tools - # curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin - - # # Update grype vulnerability database - # grype db update - - # # Check SQLite vulnerabilities in installed packages - # echo "Scanning SQLite packages for vulnerabilities..." - # GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) - # echo "Raw grype output:" - # echo "$GRYPE_RAW_OUTPUT" - - # SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) - - # if [ -n "$SQLITE_SCAN_OUTPUT" ]; then - # echo "SQLite vulnerabilities found in packages! Build should be reviewed." - # echo "SQLite vulnerability details:" - # echo "$SQLITE_SCAN_OUTPUT" - # else - # echo "No SQLite vulnerabilities found" - # fi - # continue-on-error: false - # - name: Setup pyenv - # run: | - # curl https://pyenv.run | bash - # export PATH="$HOME/.pyenv/bin:$PATH" - # eval "$(pyenv init -)" - # pyenv install 3.8:latest - # pyenv install 3.9:latest - # pyenv install 3.10:latest - # pyenv install 3.11:latest - # pyenv install 3.12:latest - # pyenv install 3.13:latest - # pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 - - # # Verify installations - # echo "Installed versions:" - # pyenv versions - # - name: Verify pyenv installations - # run: | - # export PATH="$HOME/.pyenv/bin:$PATH" - # eval "$(pyenv init -)" - # echo "Installed Python versions:" - # pyenv versions - # echo "" - # echo "Verifying all required Python versions are available:" - # for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - # if ! pyenv versions --bare | grep -q "^$version"; then - # echo "ERROR: Python $version is not installed!" - # exit 1 - # fi - # echo "✓ Python $version is installed" - # done - # echo "All Python versions verified successfully!" - # - name: Install dependencies for all Python versions - # run: | - # curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable - # source $HOME/.cargo/env - # rustup toolchain install nightly-2025-07-07 - # rustup component add --toolchain nightly-2025-07-07 rust-src - # rustc --version - # cargo --version - # export PATH="$HOME/.pyenv/bin:$PATH" - # eval "$(pyenv init -)" - # for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - # echo "Installing dependencies for Python $version" - # pyenv shell $version - # python -m pip install --upgrade pip - # if [ "$version" = "3.8" ]; then - # python -m pip install setuptools tox twine psutil wheel - # else - # python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel - # fi - # pyenv shell --unset - # done - # - name: Install clang++ for Alpine - # run: | - # apk add --no-cache make cmake ccache ninja yasm gawk wget - # apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 - # ccache -s - # - name: Update git - # run: | - # apk add --no-cache git - # git --version # - uses: taiki-e/checkout-action@v1 # with: # fetch-depth: 0 + # - name: Configure git safe directory # run: | # git config --global --add safe.directory '*' + # - name: Update submodules # run: | # git submodule update --init --recursive --jobs 4 - # # - name: ccache - # # uses: hendrikmuhs/ccache-action@v1.2 - # # with: - # # key: musllinux-1-2-aarch64 - # # max-size: 5G - # # append-timestamp: true - # - name: setup clang and link clang-20 to clang - # run: | - # ln -sf /usr/bin/clang-20 /usr/bin/clang - # ln -sf /usr/bin/clang++-20 /usr/bin/clang++ - # which clang++ - # clang++ --version - # - name: Run chdb/build-musl.sh - # timeout-minutes: 600 - # run: | - # export PATH="$HOME/.pyenv/bin:$PATH" - # eval "$(pyenv init -)" - # # Setup Rust environment - # source $HOME/.cargo/env - # pyenv shell 3.8 - # export CC=/usr/bin/clang - # export CXX=/usr/bin/clang++ - # bash ./chdb/build-musl.sh - # pyenv shell 3.9 - # bash -x ./chdb/test_smoke.sh - # continue-on-error: false - # - name: Scan chdb libraries with grype - # run: | - # echo "Scanning chdb libraries for vulnerabilities..." - - # FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" - - # SQLITE_VULNERABILITIES_FOUND=false - - # for file in $FILES_TO_SCAN; do - # if [ -f "$file" ]; then - # echo "=== Scanning $file ===" - # SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) - # echo "$SCAN_OUTPUT" - - # if echo "$SCAN_OUTPUT" | grep -qi sqlite; then - # echo "SQLite vulnerability found in $file" - # SQLITE_VULNERABILITIES_FOUND=true - # fi - # fi - # done - - # if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then - # echo "SQLite vulnerabilities detected in chdb libraries!" - # else - # echo "No SQLite vulnerabilities found in chdb libraries" - # fi - # continue-on-error: false - # - name: Check ccache statistics - # run: | - # ccache -s - # ls -lh chdb - # df -h - # - name: Build wheels - # run: | - # export PATH="$HOME/.pyenv/bin:$PATH" - # eval "$(pyenv init -)" - # export CC=/usr/bin/clang - # export CXX=/usr/bin/clang++ - # pyenv shell 3.8 - # make wheel - # - name: Install patchelf from github - # run: | - # wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz - # tar -xvf patchelf.tar.gz - # cp bin/patchelf /usr/bin/ - # chmod +x /usr/bin/patchelf - # patchelf --version - # - name: Audit wheels - # run: | - # export PATH="$HOME/.pyenv/bin:$PATH" - # eval "$(pyenv init -)" - # pyenv shell 3.13 - # python -m pip install auditwheel - # auditwheel -v repair -w dist/ --plat musllinux_1_2_aarch64 dist/*.whl - # continue-on-error: false - # - name: Show files - # run: | - # rm -f dist/*-linux_aarch64.whl - # ls -lh dist - # shell: bash - # - name: Test wheel on all Python versions - # run: | - # export PATH="$HOME/.pyenv/bin:$PATH" - # eval "$(pyenv init -)" - # for version in 3.9 3.10 3.11 3.12 3.13; do - # echo "Testing chdb on Python $version" - # pyenv shell $version - # python -m pip install dist/*.whl --force-reinstall - # python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" - # make test - # pyenv shell --unset - # done - # continue-on-error: false - # - name: Upload wheels to release - # if: startsWith(github.ref, 'refs/tags/v') - # run: | - # gh release upload ${{ github.ref_name }} dist/*.whl --clobber - # env: - # GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Create test file and copy artifacts - run: | - # mkdir -p /host-artifacts - # cp ./dist/*.whl /host-artifacts/ - # ls -la /host-artifacts/ - echo "Test file created and artifacts copied" > /host-artifacts/test.whl - # - name: Upload pypi - # if: startsWith(github.ref, 'refs/tags/v') - # run: | - # export PATH="$HOME/.pyenv/bin:$PATH" - # eval "$(pyenv init -)" - # pyenv shell 3.13 - # python -m twine upload dist/*.whl - # env: - # TWINE_USERNAME: __token__ - # TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - - upload_artifacts: - name: Upload build artifacts - runs-on: GH-Linux-ARM64 - needs: [prepare_host, build_musllinux_wheels] - steps: - - name: Check artifacts directory + + - name: Build chdb wheels in container + uses: addnab/docker-run-action@v3 + with: + image: quay.io/pypa/musllinux_1_2_aarch64 + options: -v ${{ github.workspace }}:/workspace --privileged + run: | + cd /workspace + + echo "awq" > ./acd.txt + + # # 1. Check system info + # echo "=== Container System Info ===" + # echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" + # if [ -f /lib/ld-musl-aarch64.so.1 ]; then + # echo "musl libc aarch64" + # elif [ -f /lib/libc.musl-aarch64.so.1 ]; then + # echo "musl libc aarch64" + # else + # echo "Not musl libc" + # fi + # echo "Workspace mounted at: /workspace" + # ls -la /workspace + + # # 2. Install build dependencies + # echo "=== Installing build dependencies ===" + # apk update + # apk add --no-cache make build-base openssl-dev zlib-dev \ + # bzip2-dev readline-dev sqlite-dev wget curl llvm \ + # ncurses-dev xz-dev tk-dev libxml2-dev \ + # libffi-dev linux-headers git + # apk add --no-cache make cmake ccache ninja yasm gawk + # apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 + + # # 3. Scan SQLite vulnerabilities + # echo "=== Scanning SQLite vulnerabilities ===" + # # Install grype + # curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin + # grype db update + + # # Check SQLite vulnerabilities + # echo "Scanning SQLite packages for vulnerabilities..." + # GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) + # echo "Raw grype output:" + # echo "$GRYPE_RAW_OUTPUT" + + # SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) + # if [ -n "$SQLITE_SCAN_OUTPUT" ]; then + # echo "SQLite vulnerabilities found in packages! Build should be reviewed." + # echo "SQLite vulnerability details:" + # echo "$SQLITE_SCAN_OUTPUT" + # else + # echo "No SQLite vulnerabilities found" + # fi + + # # 4. Setup Python environments + # echo "=== Setting up Python environments ===" + # # Setup pyenv + # curl https://pyenv.run | bash + # export PATH="$HOME/.pyenv/bin:$PATH" + # eval "$(pyenv init -)" + + # # Install Python versions + # for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + # echo "Installing Python $version" + # pyenv install $version:latest + # done + # pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 + + # # Verify installations + # echo "Installed Python versions:" + # pyenv versions + + # # Install Rust + # curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + # source $HOME/.cargo/env + # rustup toolchain install nightly-2025-07-07 + # rustup component add --toolchain nightly-2025-07-07 rust-src + + # # Install Python dependencies + # for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + # echo "Installing dependencies for Python $version" + # pyenv shell $version + # python -m pip install --upgrade pip + # if [ "$version" = "3.8" ]; then + # python -m pip install setuptools tox twine psutil wheel + # else + # python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel + # fi + # pyenv shell --unset + # done + + # # 5. Build chdb + # echo "=== Building chdb ===" + # # Setup clang + # ln -sf /usr/bin/clang-20 /usr/bin/clang + # ln -sf /usr/bin/clang++-20 /usr/bin/clang++ + # export CC=/usr/bin/clang + # export CXX=/usr/bin/clang++ + + # # Build + # pyenv shell 3.8 + # bash ./chdb/build-musl.sh + + # # Test + # pyenv shell 3.9 + # bash -x ./chdb/test_smoke.sh + + # # Check build results + # ccache -s + # ls -lh chdb + + # # 6. Create and audit wheels + # echo "=== Creating and auditing wheels ===" + # # Build wheels + # pyenv shell 3.8 + # make wheel + + # # Install patchelf + # wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz + # tar -xvf patchelf.tar.gz + # cp bin/patchelf /usr/bin/ + # chmod +x /usr/bin/patchelf + + # # Audit wheels + # pyenv shell 3.13 + # python -m pip install auditwheel + # auditwheel -v repair -w dist/ --plat musllinux_1_2_aarch64 dist/*.whl + + # # Clean up non-musllinux wheels + # rm -f dist/*-linux_aarch64.whl + # ls -lh dist + + # # 7. Test wheels + # echo "=== Testing wheels ===" + # for version in 3.9 3.10 3.11 3.12 3.13; do + # echo "Testing chdb on Python $version" + # pyenv shell $version + # python -m pip install dist/*.whl --force-reinstall + # python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" + # make test + # pyenv shell --unset + # done + + # # 8. Scan chdb libraries + # echo "=== Scanning chdb libraries ===" + # FILES_TO_SCAN="$(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" + # SQLITE_VULNERABILITIES_FOUND=false + + # for file in $FILES_TO_SCAN; do + # if [ -f "$file" ]; then + # echo "=== Scanning $file ===" + # SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) + # echo "$SCAN_OUTPUT" + + # if echo "$SCAN_OUTPUT" | grep -qi sqlite; then + # echo "SQLite vulnerability found in $file" + # SQLITE_VULNERABILITIES_FOUND=true + # fi + # fi + # done + + # if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then + # echo "SQLite vulnerabilities detected in chdb libraries!" + # else + # echo "No SQLite vulnerabilities found in chdb libraries" + # fi + + # # Show final results + # echo "=== Final wheel files ===" + # ls -la ./dist/ + + # Upload wheels to release + - name: Upload wheels to release + if: startsWith(github.ref, 'refs/tags/v') run: | - ARTIFACTS_DIR="${{ needs.prepare_host.outputs.artifacts_dir }}" - echo "Checking artifacts in directory: $ARTIFACTS_DIR" - ls -la "$ARTIFACTS_DIR" || echo "No artifacts directory found" - - name: Upload artifacts + echo "=== Uploading wheels to release ===" + ls -la ./dist/ + gh release upload ${{ github.ref_name }} ./dist/*.whl --clobber + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + + # Upload to PyPI + - name: Upload to PyPI + if: startsWith(github.ref, 'refs/tags/v') + uses: addnab/docker-run-action@v3 + with: + image: quay.io/pypa/musllinux_1_2_aarch64 + options: -v ${{ github.workspace }}:/workspace + run: | + cd /workspace + echo "=== Uploading to PyPI ===" + + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv shell 3.13 + python -m twine upload ./dist/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + + # Upload artifacts + - name: Upload build artifacts uses: actions/upload-artifact@v4 with: - name: chdb-artifacts-musllinux-aarch64-${{ github.run_id }} + name: chdb-artifacts-musllinux-aarch64 path: | - ${{ needs.prepare_host.outputs.artifacts_dir }}/*.whl + # ./dist/*.whl + ./acd.txt overwrite: true - - cleanup: - name: Cleanup temporary directories - runs-on: GH-Linux-ARM64 - needs: [prepare_host, build_musllinux_wheels, upload_artifacts] - if: always() - steps: - - name: Remove temporary artifacts directory - run: | - ARTIFACTS_DIR="${{ needs.prepare_host.outputs.artifacts_dir }}" - echo "Cleaning up temporary directory: $ARTIFACTS_DIR" - - if [ -d "$ARTIFACTS_DIR" ]; then - echo "Directory contents before cleanup:" - ls -la "$ARTIFACTS_DIR" - - sudo rm -rf "$ARTIFACTS_DIR" - echo "Successfully removed $ARTIFACTS_DIR" - else - echo "Directory $ARTIFACTS_DIR not found, nothing to clean" - fi - - if [ ! -d "$ARTIFACTS_DIR" ]; then - echo "✅ Cleanup successful - directory removed" - else - echo "❌ Cleanup failed - directory still exists" - exit 1 - fi From 0c5044bdd684bfc02addf6869464607d72c1eb52 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Tue, 14 Oct 2025 12:50:27 +0800 Subject: [PATCH 15/41] test: fix tests --- .../build_musllinux_arm64_wheels.yml | 343 +++++++++--------- 1 file changed, 170 insertions(+), 173 deletions(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index cdd9c687f23..6a5409e1720 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -24,17 +24,17 @@ jobs: name: Build musllinux wheels (Alpine Linux aarch64) runs-on: GH-Linux-ARM64 steps: - # - uses: taiki-e/checkout-action@v1 - # with: - # fetch-depth: 0 + - uses: taiki-e/checkout-action@v1 + with: + fetch-depth: 0 - # - name: Configure git safe directory - # run: | - # git config --global --add safe.directory '*' + - name: Configure git safe directory + run: | + git config --global --add safe.directory '*' - # - name: Update submodules - # run: | - # git submodule update --init --recursive --jobs 4 + - name: Update submodules + run: | + git submodule update --init --recursive --jobs 4 - name: Build chdb wheels in container uses: addnab/docker-run-action@v3 @@ -44,168 +44,166 @@ jobs: run: | cd /workspace - echo "awq" > ./acd.txt - - # # 1. Check system info - # echo "=== Container System Info ===" - # echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" - # if [ -f /lib/ld-musl-aarch64.so.1 ]; then - # echo "musl libc aarch64" - # elif [ -f /lib/libc.musl-aarch64.so.1 ]; then - # echo "musl libc aarch64" - # else - # echo "Not musl libc" - # fi - # echo "Workspace mounted at: /workspace" - # ls -la /workspace - - # # 2. Install build dependencies - # echo "=== Installing build dependencies ===" - # apk update - # apk add --no-cache make build-base openssl-dev zlib-dev \ - # bzip2-dev readline-dev sqlite-dev wget curl llvm \ - # ncurses-dev xz-dev tk-dev libxml2-dev \ - # libffi-dev linux-headers git - # apk add --no-cache make cmake ccache ninja yasm gawk - # apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 - - # # 3. Scan SQLite vulnerabilities - # echo "=== Scanning SQLite vulnerabilities ===" - # # Install grype - # curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin - # grype db update - - # # Check SQLite vulnerabilities - # echo "Scanning SQLite packages for vulnerabilities..." - # GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) - # echo "Raw grype output:" - # echo "$GRYPE_RAW_OUTPUT" - - # SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) - # if [ -n "$SQLITE_SCAN_OUTPUT" ]; then - # echo "SQLite vulnerabilities found in packages! Build should be reviewed." - # echo "SQLite vulnerability details:" - # echo "$SQLITE_SCAN_OUTPUT" - # else - # echo "No SQLite vulnerabilities found" - # fi - - # # 4. Setup Python environments - # echo "=== Setting up Python environments ===" - # # Setup pyenv - # curl https://pyenv.run | bash - # export PATH="$HOME/.pyenv/bin:$PATH" - # eval "$(pyenv init -)" - - # # Install Python versions - # for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - # echo "Installing Python $version" - # pyenv install $version:latest - # done - # pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 - - # # Verify installations - # echo "Installed Python versions:" - # pyenv versions - - # # Install Rust - # curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable - # source $HOME/.cargo/env - # rustup toolchain install nightly-2025-07-07 - # rustup component add --toolchain nightly-2025-07-07 rust-src - - # # Install Python dependencies - # for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - # echo "Installing dependencies for Python $version" - # pyenv shell $version - # python -m pip install --upgrade pip - # if [ "$version" = "3.8" ]; then - # python -m pip install setuptools tox twine psutil wheel - # else - # python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel - # fi - # pyenv shell --unset - # done - - # # 5. Build chdb - # echo "=== Building chdb ===" - # # Setup clang - # ln -sf /usr/bin/clang-20 /usr/bin/clang - # ln -sf /usr/bin/clang++-20 /usr/bin/clang++ - # export CC=/usr/bin/clang - # export CXX=/usr/bin/clang++ - - # # Build - # pyenv shell 3.8 - # bash ./chdb/build-musl.sh - - # # Test - # pyenv shell 3.9 - # bash -x ./chdb/test_smoke.sh - - # # Check build results - # ccache -s - # ls -lh chdb - - # # 6. Create and audit wheels - # echo "=== Creating and auditing wheels ===" - # # Build wheels - # pyenv shell 3.8 - # make wheel - - # # Install patchelf - # wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz - # tar -xvf patchelf.tar.gz - # cp bin/patchelf /usr/bin/ - # chmod +x /usr/bin/patchelf - - # # Audit wheels - # pyenv shell 3.13 - # python -m pip install auditwheel - # auditwheel -v repair -w dist/ --plat musllinux_1_2_aarch64 dist/*.whl - - # # Clean up non-musllinux wheels - # rm -f dist/*-linux_aarch64.whl - # ls -lh dist - - # # 7. Test wheels - # echo "=== Testing wheels ===" - # for version in 3.9 3.10 3.11 3.12 3.13; do - # echo "Testing chdb on Python $version" - # pyenv shell $version - # python -m pip install dist/*.whl --force-reinstall - # python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" - # make test - # pyenv shell --unset - # done - - # # 8. Scan chdb libraries - # echo "=== Scanning chdb libraries ===" - # FILES_TO_SCAN="$(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" - # SQLITE_VULNERABILITIES_FOUND=false - - # for file in $FILES_TO_SCAN; do - # if [ -f "$file" ]; then - # echo "=== Scanning $file ===" - # SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) - # echo "$SCAN_OUTPUT" - - # if echo "$SCAN_OUTPUT" | grep -qi sqlite; then - # echo "SQLite vulnerability found in $file" - # SQLITE_VULNERABILITIES_FOUND=true - # fi - # fi - # done - - # if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then - # echo "SQLite vulnerabilities detected in chdb libraries!" - # else - # echo "No SQLite vulnerabilities found in chdb libraries" - # fi - - # # Show final results - # echo "=== Final wheel files ===" - # ls -la ./dist/ + # 1. Check system info + echo "=== Container System Info ===" + echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" + if [ -f /lib/ld-musl-aarch64.so.1 ]; then + echo "musl libc aarch64" + elif [ -f /lib/libc.musl-aarch64.so.1 ]; then + echo "musl libc aarch64" + else + echo "Not musl libc" + fi + echo "Workspace mounted at: /workspace" + ls -la /workspace + + # 2. Install build dependencies + echo "=== Installing build dependencies ===" + apk update + apk add --no-cache make build-base openssl-dev zlib-dev \ + bzip2-dev readline-dev sqlite-dev wget curl llvm \ + ncurses-dev xz-dev tk-dev libxml2-dev \ + libffi-dev linux-headers git + apk add --no-cache make cmake ccache ninja yasm gawk + apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 + + # 3. Scan SQLite vulnerabilities + echo "=== Scanning SQLite vulnerabilities ===" + # Install grype + curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin + grype db update + + # Check SQLite vulnerabilities + echo "Scanning SQLite packages for vulnerabilities..." + GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) + echo "Raw grype output:" + echo "$GRYPE_RAW_OUTPUT" + + SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) + if [ -n "$SQLITE_SCAN_OUTPUT" ]; then + echo "SQLite vulnerabilities found in packages! Build should be reviewed." + echo "SQLite vulnerability details:" + echo "$SQLITE_SCAN_OUTPUT" + else + echo "No SQLite vulnerabilities found" + fi + + # 4. Setup Python environments + echo "=== Setting up Python environments ===" + # Setup pyenv + curl https://pyenv.run | bash + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + + # Install Python versions + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + echo "Installing Python $version" + pyenv install $version:latest + done + pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 + + # Verify installations + echo "Installed Python versions:" + pyenv versions + + # Install Rust + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + source $HOME/.cargo/env + rustup toolchain install nightly-2025-07-07 + rustup component add --toolchain nightly-2025-07-07 rust-src + + # Install Python dependencies + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + echo "Installing dependencies for Python $version" + pyenv shell $version + python -m pip install --upgrade pip + if [ "$version" = "3.8" ]; then + python -m pip install setuptools tox twine psutil wheel + else + python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel + fi + pyenv shell --unset + done + + # 5. Build chdb + echo "=== Building chdb ===" + # Setup clang + ln -sf /usr/bin/clang-20 /usr/bin/clang + ln -sf /usr/bin/clang++-20 /usr/bin/clang++ + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + + # Build + pyenv shell 3.8 + bash ./chdb/build-musl.sh + + # Test + pyenv shell 3.9 + bash -x ./chdb/test_smoke.sh + + # Check build results + ccache -s + ls -lh chdb + + # 6. Create and audit wheels + echo "=== Creating and auditing wheels ===" + # Build wheels + pyenv shell 3.8 + make wheel + + # Install patchelf + wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz + tar -xvf patchelf.tar.gz + cp bin/patchelf /usr/bin/ + chmod +x /usr/bin/patchelf + + # Audit wheels + pyenv shell 3.13 + python -m pip install auditwheel + auditwheel -v repair -w dist/ --plat musllinux_1_2_aarch64 dist/*.whl + + # Clean up non-musllinux wheels + rm -f dist/*-linux_aarch64.whl + ls -lh dist + + # 7. Test wheels + echo "=== Testing wheels ===" + for version in 3.9 3.10 3.11 3.12 3.13; do + echo "Testing chdb on Python $version" + pyenv shell $version + python -m pip install dist/*.whl --force-reinstall + python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" + make test + pyenv shell --unset + done + + # 8. Scan chdb libraries + echo "=== Scanning chdb libraries ===" + FILES_TO_SCAN="$(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" + SQLITE_VULNERABILITIES_FOUND=false + + for file in $FILES_TO_SCAN; do + if [ -f "$file" ]; then + echo "=== Scanning $file ===" + SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) + echo "$SCAN_OUTPUT" + + if echo "$SCAN_OUTPUT" | grep -qi sqlite; then + echo "SQLite vulnerability found in $file" + SQLITE_VULNERABILITIES_FOUND=true + fi + fi + done + + if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then + echo "SQLite vulnerabilities detected in chdb libraries!" + else + echo "No SQLite vulnerabilities found in chdb libraries" + fi + + # Show final results + echo "=== Final wheel files ===" + ls -la ./dist/ # Upload wheels to release - name: Upload wheels to release @@ -242,6 +240,5 @@ jobs: with: name: chdb-artifacts-musllinux-aarch64 path: | - # ./dist/*.whl - ./acd.txt + ./dist/*.whl overwrite: true From 7a55c0e2a3ef3e62a093e087368ecf0d18d47be1 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Tue, 14 Oct 2025 13:00:25 +0800 Subject: [PATCH 16/41] chore: update build_musllinux_arm64_wheels.yml --- .github/workflows/build_musllinux_arm64_wheels.yml | 10 +++++++++- .github/workflows/build_musllinux_x86_wheels.yml | 5 +---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index 6a5409e1720..0b26d724740 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -103,8 +103,16 @@ jobs: pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 # Verify installations - echo "Installed Python versions:" + echo "Installed versions:" pyenv versions + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + if ! pyenv versions --bare | grep -q "^$version"; then + echo "ERROR: Python $version is not installed!" + exit 1 + fi + echo "Python $version is installed" + done + echo "All Python versions verified successfully!" # Install Rust curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable diff --git a/.github/workflows/build_musllinux_x86_wheels.yml b/.github/workflows/build_musllinux_x86_wheels.yml index bfaabea2f8f..c48a803f97c 100644 --- a/.github/workflows/build_musllinux_x86_wheels.yml +++ b/.github/workflows/build_musllinux_x86_wheels.yml @@ -89,16 +89,13 @@ jobs: run: | export PATH="$HOME/.pyenv/bin:$PATH" eval "$(pyenv init -)" - echo "Installed Python versions:" - pyenv versions - echo "" echo "Verifying all required Python versions are available:" for version in 3.8 3.9 3.10 3.11 3.12 3.13; do if ! pyenv versions --bare | grep -q "^$version"; then echo "ERROR: Python $version is not installed!" exit 1 fi - echo "✓ Python $version is installed" + echo "Python $version is installed" done echo "All Python versions verified successfully!" - name: Install dependencies for all Python versions From b69e59561fdeba5a4cf3b03eda15dd35ff996d16 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Tue, 14 Oct 2025 14:22:12 +0800 Subject: [PATCH 17/41] chore: update build_musllinux_arm64_wheels.yml --- .../build_musllinux_arm64_wheels.yml | 80 ++++++++++++++++++- 1 file changed, 78 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index 0b26d724740..1d22347e185 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -44,6 +44,9 @@ jobs: run: | cd /workspace + # Configure git safe directory in container + git config --global --add safe.directory /workspace + # 1. Check system info echo "=== Container System Info ===" echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" @@ -135,56 +138,129 @@ jobs: # 5. Build chdb echo "=== Building chdb ===" + echo "Timestamp: $(date)" + echo "Current directory: $(pwd)" + echo "Available disk space: $(df -h .)" + # Setup clang + echo "Setting up clang compiler..." ln -sf /usr/bin/clang-20 /usr/bin/clang ln -sf /usr/bin/clang++-20 /usr/bin/clang++ export CC=/usr/bin/clang export CXX=/usr/bin/clang++ + echo "Compiler versions:" + $CC --version + $CXX --version # Build + echo "Starting chdb build with Python 3.8..." pyenv shell 3.8 + python --version + echo "Build start time: $(date)" bash ./chdb/build-musl.sh + echo "Build end time: $(date)" # Test + echo "Running smoke test with Python 3.9..." pyenv shell 3.9 + python --version + echo "Test start time: $(date)" bash -x ./chdb/test_smoke.sh + echo "Test end time: $(date)" # Check build results + echo "Build results summary:" ccache -s + echo "chdb directory contents:" ls -lh chdb + echo "Build artifacts size:" + du -sh chdb # 6. Create and audit wheels echo "=== Creating and auditing wheels ===" + echo "Wheel creation start time: $(date)" + echo "Available disk space before wheel build: $(df -h .)" + # Build wheels + echo "Building wheels with Python 3.8..." pyenv shell 3.8 + python --version + echo "Running make wheel..." make wheel + echo "Wheel build completed at: $(date)" + echo "Initial wheel files:" + ls -lh dist/ || echo "No dist directory yet" # Install patchelf + echo "Installing patchelf for wheel auditing..." wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz tar -xvf patchelf.tar.gz cp bin/patchelf /usr/bin/ chmod +x /usr/bin/patchelf + echo "patchelf version: $(patchelf --version)" # Audit wheels + echo "Auditing wheels with Python 3.13..." pyenv shell 3.13 + python --version python -m pip install auditwheel + echo "auditwheel version: $(auditwheel --version)" + echo "Starting wheel audit at: $(date)" auditwheel -v repair -w dist/ --plat musllinux_1_2_aarch64 dist/*.whl + echo "Wheel audit completed at: $(date)" # Clean up non-musllinux wheels + echo "Cleaning up non-musllinux wheels..." + echo "Before cleanup:" + ls -lh dist/ rm -f dist/*-linux_aarch64.whl - ls -lh dist + echo "After cleanup:" + ls -lh dist/ + echo "Final wheel sizes:" + du -sh dist/* # 7. Test wheels echo "=== Testing wheels ===" + echo "Wheel testing start time: $(date)" + echo "Available wheels for testing:" + ls -lh dist/*.whl + echo "Wheel file details:" + file dist/*.whl + + TOTAL_TESTS=5 + CURRENT_TEST=0 + for version in 3.9 3.10 3.11 3.12 3.13; do - echo "Testing chdb on Python $version" + CURRENT_TEST=$((CURRENT_TEST + 1)) + echo "=== Test $CURRENT_TEST/$TOTAL_TESTS: Python $version ===" + echo "Test start time: $(date)" + + echo "Switching to Python $version..." pyenv shell $version + python --version + echo "pip version: $(python -m pip --version)" + + echo "Installing chdb wheel..." python -m pip install dist/*.whl --force-reinstall + echo "Installation completed at: $(date)" + + echo "Verifying chdb installation..." + python -c "import chdb; print(f'chdb imported successfully'); print(f'chdb version: {getattr(chdb, '__version__', 'unknown')}')" + + echo "Running basic query test..." python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" + + echo "Running full test suite..." make test + echo "Test suite completed for Python $version at: $(date)" + pyenv shell --unset + echo "Test $CURRENT_TEST/$TOTAL_TESTS completed successfully" + echo "" done + echo "All wheel tests completed at: $(date)" + # 8. Scan chdb libraries echo "=== Scanning chdb libraries ===" FILES_TO_SCAN="$(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" From b863480527d0875d996ac7664c8e684fc2d606b9 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Tue, 14 Oct 2025 14:25:24 +0800 Subject: [PATCH 18/41] chore: update build_musllinux_arm64_wheels.yml --- .github/workflows/build_musllinux_arm64_wheels.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index 1d22347e185..d986f745797 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -44,9 +44,6 @@ jobs: run: | cd /workspace - # Configure git safe directory in container - git config --global --add safe.directory /workspace - # 1. Check system info echo "=== Container System Info ===" echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" @@ -136,6 +133,10 @@ jobs: pyenv shell --unset done + # Configure git safe directory in container + echo "=== Configure git safe directory ===" + git config --global --add safe.directory /workspace + # 5. Build chdb echo "=== Building chdb ===" echo "Timestamp: $(date)" From 8f39dd81e3843757d50edd6bf545cdec964a27af Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Tue, 14 Oct 2025 15:53:04 +0800 Subject: [PATCH 19/41] chore: update build_musllinux_arm64_wheels.yml --- .../build_musllinux_arm64_wheels.yml | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index d986f745797..b020c36f2f6 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -24,7 +24,7 @@ jobs: name: Build musllinux wheels (Alpine Linux aarch64) runs-on: GH-Linux-ARM64 steps: - - uses: taiki-e/checkout-action@v1 + - uses: actions/checkout@v3 with: fetch-depth: 0 @@ -32,9 +32,9 @@ jobs: run: | git config --global --add safe.directory '*' - - name: Update submodules - run: | - git submodule update --init --recursive --jobs 4 + # - name: Update submodules + # run: | + # git submodule update --init --recursive --jobs 4 - name: Build chdb wheels in container uses: addnab/docker-run-action@v3 @@ -44,6 +44,13 @@ jobs: run: | cd /workspace + # Configure git safe directory in container + apk update + apk add --no-cache git + echo "=== Configure git safe directory ===" + git config --global --add safe.directory /workspace + git describe --tags + # 1. Check system info echo "=== Container System Info ===" echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" @@ -59,11 +66,10 @@ jobs: # 2. Install build dependencies echo "=== Installing build dependencies ===" - apk update apk add --no-cache make build-base openssl-dev zlib-dev \ bzip2-dev readline-dev sqlite-dev wget curl llvm \ ncurses-dev xz-dev tk-dev libxml2-dev \ - libffi-dev linux-headers git + libffi-dev linux-headers apk add --no-cache make cmake ccache ninja yasm gawk apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 @@ -133,10 +139,6 @@ jobs: pyenv shell --unset done - # Configure git safe directory in container - echo "=== Configure git safe directory ===" - git config --global --add safe.directory /workspace - # 5. Build chdb echo "=== Building chdb ===" echo "Timestamp: $(date)" From aa4493edfc04ae63330be0510b818370f936bf1f Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Tue, 14 Oct 2025 15:56:46 +0800 Subject: [PATCH 20/41] chore: update build_musllinux_arm64_wheels.yml --- .github/workflows/build_musllinux_arm64_wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index b020c36f2f6..e0b7d253733 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -50,6 +50,7 @@ jobs: echo "=== Configure git safe directory ===" git config --global --add safe.directory /workspace git describe --tags + python3 -c "import sys; sys.path.append('.'); from setup import get_latest_git_tag; print('setup.get_latest_git_tag():', get_latest_git_tag())" # 1. Check system info echo "=== Container System Info ===" From 74aa4f48b90dc82301f65b15aeb1890930f61148 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Tue, 14 Oct 2025 16:01:12 +0800 Subject: [PATCH 21/41] chore: update build_musllinux_arm64_wheels.yml --- .github/workflows/build_musllinux_arm64_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index e0b7d253733..5fb0fbee8b4 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -46,7 +46,7 @@ jobs: # Configure git safe directory in container apk update - apk add --no-cache git + apk add --no-cache git python3 py3-pip py3-setuptools echo "=== Configure git safe directory ===" git config --global --add safe.directory /workspace git describe --tags From 733e9824dfa80fd5afc9f81e2366b7e9e470e944 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Tue, 14 Oct 2025 16:05:02 +0800 Subject: [PATCH 22/41] chore: update build_musllinux_arm64_wheels.yml --- .github/workflows/build_musllinux_arm64_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index 5fb0fbee8b4..75201ea1df0 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -24,7 +24,7 @@ jobs: name: Build musllinux wheels (Alpine Linux aarch64) runs-on: GH-Linux-ARM64 steps: - - uses: actions/checkout@v3 + - uses: taiki-e/checkout-action@v1 with: fetch-depth: 0 From 00e8a0fc39e24b0be752eac6feab0bee54f31b20 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Tue, 14 Oct 2025 16:07:33 +0800 Subject: [PATCH 23/41] chore: update build_musllinux_arm64_wheels.yml --- .github/workflows/build_musllinux_arm64_wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index 75201ea1df0..65b853f383d 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -24,7 +24,7 @@ jobs: name: Build musllinux wheels (Alpine Linux aarch64) runs-on: GH-Linux-ARM64 steps: - - uses: taiki-e/checkout-action@v1 + - uses: actions/checkout@v3 with: fetch-depth: 0 @@ -50,7 +50,7 @@ jobs: echo "=== Configure git safe directory ===" git config --global --add safe.directory /workspace git describe --tags - python3 -c "import sys; sys.path.append('.'); from setup import get_latest_git_tag; print('setup.get_latest_git_tag():', get_latest_git_tag())" + python3 -c "import sys; sys.path.append('.'); from setup import get_latest_git_tag; print('version:', get_latest_git_tag())" # 1. Check system info echo "=== Container System Info ===" From 3c7227e34581a757f3c9614233702749375e406b Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Tue, 14 Oct 2025 17:36:10 +0800 Subject: [PATCH 24/41] chore: fix musl arm64 build --- .github/workflows/build_musllinux_arm64_wheels.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index 65b853f383d..5e5e63cc1b6 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -32,9 +32,9 @@ jobs: run: | git config --global --add safe.directory '*' - # - name: Update submodules - # run: | - # git submodule update --init --recursive --jobs 4 + - name: Update submodules + run: | + git submodule update --init --recursive --jobs 4 - name: Build chdb wheels in container uses: addnab/docker-run-action@v3 @@ -292,7 +292,7 @@ jobs: # Show final results echo "=== Final wheel files ===" ls -la ./dist/ - + continue-on-error: false # Upload wheels to release - name: Upload wheels to release if: startsWith(github.ref, 'refs/tags/v') From 1f16f3c575491a0acb53ba7a3e364c022e6f6200 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Tue, 14 Oct 2025 17:39:59 +0800 Subject: [PATCH 25/41] fix: fix musl arm64 build --- .github/workflows/build_musllinux_arm64_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index 5e5e63cc1b6..c6de317fff3 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -292,7 +292,7 @@ jobs: # Show final results echo "=== Final wheel files ===" ls -la ./dist/ - continue-on-error: false + continue-on-error: false # Upload wheels to release - name: Upload wheels to release if: startsWith(github.ref, 'refs/tags/v') From ed6b93f262b746022a88c5687fa7ced10fdf8541 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Tue, 14 Oct 2025 19:56:11 +0800 Subject: [PATCH 26/41] fix: fix musl arm64 build --- .github/workflows/build_musllinux_arm64_wheels.yml | 1 + PreLoad.cmake | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index c6de317fff3..2793f44fb7f 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -43,6 +43,7 @@ jobs: options: -v ${{ github.workspace }}:/workspace --privileged run: | cd /workspace + set -e # Configure git safe directory in container apk update diff --git a/PreLoad.cmake b/PreLoad.cmake index 0e13f3b541c..4672f0cb5c6 100644 --- a/PreLoad.cmake +++ b/PreLoad.cmake @@ -95,7 +95,9 @@ if (OS MATCHES "Linux" set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-x86_64.cmake" CACHE INTERNAL "") endif() elseif (ARCH MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)") - set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-aarch64.cmake" CACHE INTERNAL "") + if ("$ENV{USE_MUSL}" STREQUAL "") + set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-aarch64.cmake" CACHE INTERNAL "") + endif() elseif (ARCH MATCHES "^(ppc64le.*|PPC64LE.*)") set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-ppc64le.cmake" CACHE INTERNAL "") elseif (ARCH MATCHES "^(s390x.*|S390X.*)") From 7d161ade47ea62911d39dd3a5d7e27951c5a2037 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Tue, 14 Oct 2025 20:03:33 +0800 Subject: [PATCH 27/41] fix: fix musl arm64 build --- .../build_musllinux_arm64_wheels.yml | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index 2793f44fb7f..9ef881454c6 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -43,7 +43,6 @@ jobs: options: -v ${{ github.workspace }}:/workspace --privileged run: | cd /workspace - set -e # Configure git safe directory in container apk update @@ -234,6 +233,7 @@ jobs: TOTAL_TESTS=5 CURRENT_TEST=0 + TEST_FAILED=false for version in 3.9 3.10 3.11 3.12 3.13; do CURRENT_TEST=$((CURRENT_TEST + 1)) @@ -256,8 +256,13 @@ jobs: python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" echo "Running full test suite..." - make test - echo "Test suite completed for Python $version at: $(date)" + if make test; then + echo "Test suite PASSED for Python $version at: $(date)" + else + echo "Test suite FAILED for Python $version at: $(date)" + TEST_FAILED=true + break + fi pyenv shell --unset echo "Test $CURRENT_TEST/$TOTAL_TESTS completed successfully" @@ -266,6 +271,19 @@ jobs: echo "All wheel tests completed at: $(date)" + # Check if any tests failed + if [ "$TEST_FAILED" = true ]; then + echo "ERROR: One or more test suites failed!" + echo "Test failure detected - aborting build process" + exit 1 + fi + + # Create test success marker file only if all tests passed + echo "All tests passed successfully!" + echo "Creating test success marker..." + touch /workspace/.test_success_marker + echo "Test success marker created at: $(date)" + # 8. Scan chdb libraries echo "=== Scanning chdb libraries ===" FILES_TO_SCAN="$(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" @@ -294,6 +312,18 @@ jobs: echo "=== Final wheel files ===" ls -la ./dist/ continue-on-error: false + # Check test success before upload + - name: Verify test completion + run: | + echo "=== Verifying test completion ===" + if [ ! -f ".test_success_marker" ]; then + echo "ERROR: Test success marker file not found!" + echo "This indicates that the wheel testing did not complete successfully." + echo "Aborting upload process." + exit 1 + fi + echo "Test success marker found. All tests completed successfully." + echo "Proceeding with wheel upload..." # Upload wheels to release - name: Upload wheels to release if: startsWith(github.ref, 'refs/tags/v') From 5a6098420e4f27ae61f2be21534f6b625f2e823b Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Tue, 14 Oct 2025 20:05:32 +0800 Subject: [PATCH 28/41] test: add check for musl arm test --- .github/workflows/build_musllinux_arm64_wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index 9ef881454c6..988c6c586fc 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -324,6 +324,7 @@ jobs: fi echo "Test success marker found. All tests completed successfully." echo "Proceeding with wheel upload..." + continue-on-error: false # Upload wheels to release - name: Upload wheels to release if: startsWith(github.ref, 'refs/tags/v') From 11227eb9ed443d4b2d4bfc730e3bded1a58e8b48 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Tue, 14 Oct 2025 22:29:26 +0800 Subject: [PATCH 29/41] chore: update build_musllinux_arm64_wheels.yml --- .../build_musllinux_arm64_wheels.yml | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index 988c6c586fc..59b6df41def 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -103,16 +103,19 @@ jobs: eval "$(pyenv init -)" # Install Python versions - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + # for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + for version in 3.8 3.9; do echo "Installing Python $version" pyenv install $version:latest done - pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 + # pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 + pyenv global 3.8 3.9 # Verify installations echo "Installed versions:" pyenv versions - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + # for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + for version in 3.8 3.9; do if ! pyenv versions --bare | grep -q "^$version"; then echo "ERROR: Python $version is not installed!" exit 1 @@ -128,14 +131,16 @@ jobs: rustup component add --toolchain nightly-2025-07-07 rust-src # Install Python dependencies - for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + # for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + for version in 3.8 3.9; do echo "Installing dependencies for Python $version" pyenv shell $version python -m pip install --upgrade pip if [ "$version" = "3.8" ]; then python -m pip install setuptools tox twine psutil wheel else - python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel + # python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel + python -m pip install setuptools tox pandas pyarrow twine psutil wheel fi pyenv shell --unset done @@ -146,15 +151,17 @@ jobs: echo "Current directory: $(pwd)" echo "Available disk space: $(df -h .)" - # Setup clang - echo "Setting up clang compiler..." + # Setup clang and lld + echo "Setting up clang compiler and lld linker..." ln -sf /usr/bin/clang-20 /usr/bin/clang ln -sf /usr/bin/clang++-20 /usr/bin/clang++ + ln -sf /usr/bin/ld.lld-20 /usr/bin/ld.lld export CC=/usr/bin/clang export CXX=/usr/bin/clang++ - echo "Compiler versions:" + echo "Compiler and linker versions:" $CC --version $CXX --version + /usr/bin/ld.lld --version # Build echo "Starting chdb build with Python 3.8..." From dc0e69a49ec66845a3e5bfbec94b65e2f1919614 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Tue, 14 Oct 2025 23:18:27 +0800 Subject: [PATCH 30/41] chore: update build_musllinux_arm64_wheels.yml --- .github/workflows/build_musllinux_arm64_wheels.yml | 8 +++----- programs/local/chdb.cpp | 11 +++++++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index 59b6df41def..324c0e2ae6e 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -151,17 +151,15 @@ jobs: echo "Current directory: $(pwd)" echo "Available disk space: $(df -h .)" - # Setup clang and lld - echo "Setting up clang compiler and lld linker..." + # Setup clang + echo "Setting up clang compiler..." ln -sf /usr/bin/clang-20 /usr/bin/clang ln -sf /usr/bin/clang++-20 /usr/bin/clang++ - ln -sf /usr/bin/ld.lld-20 /usr/bin/ld.lld export CC=/usr/bin/clang export CXX=/usr/bin/clang++ - echo "Compiler and linker versions:" + echo "Compiler versions:" $CC --version $CXX --version - /usr/bin/ld.lld --version # Build echo "Starting chdb build with Python 3.8..." diff --git a/programs/local/chdb.cpp b/programs/local/chdb.cpp index b9ac4b6437f..2989f2e5054 100644 --- a/programs/local/chdb.cpp +++ b/programs/local/chdb.cpp @@ -7,6 +7,17 @@ #include "QueryResult.h" #include "chdb-internal.h" +#if defined(USE_MUSL) && defined(__aarch64__) +void musl_compile_stub(int arg) +{ + jmp_buf buf1; + sigjmp_buf buf2; + + setjmp(buf1); + sigsetjmp(buf2, arg); +} +#endif + #if USE_PYTHON # include "FormatHelper.h" # include "PythonTableCache.h" From 969431864149efb1f4c23ad9d94eabd42be28e04 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Wed, 15 Oct 2025 00:01:11 +0800 Subject: [PATCH 31/41] chore: fix musl arm build --- programs/local/LocalServer.cpp | 10 ++++++++++ src/TableFunctions/ITableFunction.cpp | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 94c747cc7c4..c170232bc94 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -71,6 +71,16 @@ # include #endif +#if defined(USE_MUSL) && defined(__aarch64__) +void local_server_musl_compile_stub(int arg) +{ + jmp_buf buf1; + sigjmp_buf buf2; + + setjmp(buf1); + sigsetjmp(buf2, arg); +} +#endif namespace fs = std::filesystem; diff --git a/src/TableFunctions/ITableFunction.cpp b/src/TableFunctions/ITableFunction.cpp index 3f59a8c48a1..d71e1be8aaf 100644 --- a/src/TableFunctions/ITableFunction.cpp +++ b/src/TableFunctions/ITableFunction.cpp @@ -7,6 +7,16 @@ #include #include +#if defined(USE_MUSL) && defined(__aarch64__) +void i_table_function_musl_compile_stub(int arg) +{ + jmp_buf buf1; + sigjmp_buf buf2; + + setjmp(buf1); + sigsetjmp(buf2, arg); +} +#endif namespace ProfileEvents { From d686eff927e38f83b56e9355d9a095609cb318e5 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Wed, 15 Oct 2025 00:45:31 +0800 Subject: [PATCH 32/41] chore: fix musl arm build --- programs/main.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/programs/main.cpp b/programs/main.cpp index b2bc03a5928..67d270540d2 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -64,6 +64,17 @@ const char * __ubsan_default_options() #pragma clang diagnostic pop #endif +#if defined(USE_MUSL) && defined(__aarch64__) +void main_musl_compile_stub(int arg) +{ + jmp_buf buf1; + sigjmp_buf buf2; + + setjmp(buf1); + sigsetjmp(buf2, arg); +} +#endif + /// Universal executable for various clickhouse applications int mainEntryClickHouseBenchmark(int argc, char ** argv); int mainEntryClickHouseCheckMarks(int argc, char ** argv); From 471b470d2d6c702b9fabe3494caa6a63aa552982 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Wed, 15 Oct 2025 03:28:37 +0800 Subject: [PATCH 33/41] fix: fix musl arm test --- .github/workflows/build_musllinux_arm64_wheels.yml | 4 +--- contrib/pybind11-cmake/CMakeLists.txt | 6 ++++++ programs/local/chdb.cpp | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index 324c0e2ae6e..a6027b99efa 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -254,9 +254,6 @@ jobs: python -m pip install dist/*.whl --force-reinstall echo "Installation completed at: $(date)" - echo "Verifying chdb installation..." - python -c "import chdb; print(f'chdb imported successfully'); print(f'chdb version: {getattr(chdb, '__version__', 'unknown')}')" - echo "Running basic query test..." python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" @@ -361,6 +358,7 @@ jobs: # Upload artifacts - name: Upload build artifacts + if: always() uses: actions/upload-artifact@v4 with: name: chdb-artifacts-musllinux-aarch64 diff --git a/contrib/pybind11-cmake/CMakeLists.txt b/contrib/pybind11-cmake/CMakeLists.txt index c986799fd10..cf1cf3fa36b 100644 --- a/contrib/pybind11-cmake/CMakeLists.txt +++ b/contrib/pybind11-cmake/CMakeLists.txt @@ -76,6 +76,12 @@ if(Python_FOUND) target_include_directories(${PYBIND11_NONLIMITEDAPI_LIBNAME} PRIVATE ${PYBIND11_INCLUDE_DIR}) target_include_directories(${PYBIND11_NONLIMITEDAPI_LIBNAME} PRIVATE ${Python_INCLUDE_DIRS}) target_link_libraries(${PYBIND11_NONLIMITEDAPI_LIBNAME} PUBLIC ch_contrib::pybind11_stubs) + + if (USE_MUSL AND CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") + target_link_libraries(${PYBIND11_NONLIMITEDAPI_LIBNAME} PRIVATE -lgcc) + message(STATUS "Linking libgcc for ARM64+musl to provide compiler runtime symbols") + endif() + if (APPLE) target_link_options(${PYBIND11_NONLIMITEDAPI_LIBNAME} PRIVATE -undefined dynamic_lookup) else() diff --git a/programs/local/chdb.cpp b/programs/local/chdb.cpp index 2989f2e5054..889b61c85ea 100644 --- a/programs/local/chdb.cpp +++ b/programs/local/chdb.cpp @@ -8,7 +8,7 @@ #include "chdb-internal.h" #if defined(USE_MUSL) && defined(__aarch64__) -void musl_compile_stub(int arg) +void chdb_musl_compile_stub(int arg) { jmp_buf buf1; sigjmp_buf buf2; From c5f081575691cf2ce4759127943915c24aea02db Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Wed, 15 Oct 2025 04:32:17 +0800 Subject: [PATCH 34/41] chore: update pybind11-cmake/CMakeLists.txt --- contrib/pybind11-cmake/CMakeLists.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/contrib/pybind11-cmake/CMakeLists.txt b/contrib/pybind11-cmake/CMakeLists.txt index cf1cf3fa36b..398c6aa71a8 100644 --- a/contrib/pybind11-cmake/CMakeLists.txt +++ b/contrib/pybind11-cmake/CMakeLists.txt @@ -78,8 +78,9 @@ if(Python_FOUND) target_link_libraries(${PYBIND11_NONLIMITEDAPI_LIBNAME} PUBLIC ch_contrib::pybind11_stubs) if (USE_MUSL AND CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") - target_link_libraries(${PYBIND11_NONLIMITEDAPI_LIBNAME} PRIVATE -lgcc) - message(STATUS "Linking libgcc for ARM64+musl to provide compiler runtime symbols") + # Force linking entire libgcc archive to ensure __floatunsitf is included + target_link_libraries(${PYBIND11_NONLIMITEDAPI_LIBNAME} PRIVATE -Wl,--whole-archive -lgcc -Wl,--no-whole-archive) + message(STATUS "Linking whole libgcc archive for ARM64+musl to provide all compiler runtime symbols") endif() if (APPLE) From 209da3529f2959ea509d9c261e29b415a2edffba Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Wed, 15 Oct 2025 10:41:24 +0800 Subject: [PATCH 35/41] chore: remove redundant functions --- .../workflows/build_linux_arm64_wheels-gh.yml | 638 +++++++++--------- .github/workflows/build_linux_x86_wheels.yml | 636 ++++++++--------- .../workflows/build_macos_arm64_wheels.yml | 544 +++++++-------- .github/workflows/build_macos_x86_wheels.yml | 588 ++++++++-------- .../build_musllinux_arm64_wheels.yml | 15 +- contrib/pybind11-cmake/CMakeLists.txt | 1 - programs/local/LocalServer.cpp | 11 - src/TableFunctions/ITableFunction.cpp | 11 - 8 files changed, 1208 insertions(+), 1236 deletions(-) diff --git a/.github/workflows/build_linux_arm64_wheels-gh.yml b/.github/workflows/build_linux_arm64_wheels-gh.yml index 4f677959e52..fe9e78277a5 100644 --- a/.github/workflows/build_linux_arm64_wheels-gh.yml +++ b/.github/workflows/build_linux_arm64_wheels-gh.yml @@ -1,333 +1,333 @@ -# name: Build Linux arm64 +name: Build Linux arm64 -# on: -# workflow_dispatch: -# inputs: -# TAG_NAME: -# description: 'Release Version Tag' -# required: true -# release: -# types: [created] -# push: -# branches: -# - main -# paths-ignore: -# - '**/*.md' -# pull_request: -# branches: -# - main -# paths-ignore: -# - '**/*.md' +on: + workflow_dispatch: + inputs: + TAG_NAME: + description: 'Release Version Tag' + required: true + release: + types: [created] + push: + branches: + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - main + paths-ignore: + - '**/*.md' -# jobs: -# build_universal_wheel: -# name: Build Universal Wheel (Linux ARM64) -# runs-on: GH-Linux-ARM64 -# steps: -# - name: Install Python build dependencies -# run: | -# sudo apt-get update -# sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \ -# libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ -# libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \ -# libffi-dev liblzma-dev -# - name: Scan SQLite vulnerabilities with grype -# run: | -# # Install grype and required tools -# curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin -# sudo apt-get update && sudo apt-get install -y jq lsb-release +jobs: + build_universal_wheel: + name: Build Universal Wheel (Linux ARM64) + runs-on: GH-Linux-ARM64 + steps: + - name: Install Python build dependencies + run: | + sudo apt-get update + sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \ + libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ + libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \ + libffi-dev liblzma-dev + - name: Scan SQLite vulnerabilities with grype + run: | + # Install grype and required tools + curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin + sudo apt-get update && sudo apt-get install -y jq lsb-release -# # Detect OS distribution info -# DISTRO_ID=$(lsb_release -si | tr '[:upper:]' '[:lower:]') -# DISTRO_VERSION=$(lsb_release -sr) -# echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION" + # Detect OS distribution info + DISTRO_ID=$(lsb_release -si | tr '[:upper:]' '[:lower:]') + DISTRO_VERSION=$(lsb_release -sr) + echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION" -# # Update grype vulnerability database -# grype db update + # Update grype vulnerability database + grype db update -# # Check SQLite vulnerabilities in installed packages -# echo "Scanning SQLite packages for vulnerabilities..." -# GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true) -# echo "Raw grype output:" -# echo "$GRYPE_RAW_OUTPUT" + # Check SQLite vulnerabilities in installed packages + echo "Scanning SQLite packages for vulnerabilities..." + GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true) + echo "Raw grype output:" + echo "$GRYPE_RAW_OUTPUT" -# SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) + SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) -# if [ -n "$SQLITE_SCAN_OUTPUT" ]; then -# echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." -# echo "SQLite vulnerability details:" -# echo "$SQLITE_SCAN_OUTPUT" -# exit 1 -# else -# echo "✅ No SQLite vulnerabilities found" -# fi -# continue-on-error: false -# - name: Setup pyenv -# run: | -# curl https://pyenv.run | bash -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# pyenv install 3.8:latest -# pyenv install 3.9:latest -# pyenv install 3.10:latest -# pyenv install 3.11:latest -# pyenv install 3.12:latest -# pyenv install 3.13:latest -# pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 + if [ -n "$SQLITE_SCAN_OUTPUT" ]; then + echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." + echo "SQLite vulnerability details:" + echo "$SQLITE_SCAN_OUTPUT" + exit 1 + else + echo "✅ No SQLite vulnerabilities found" + fi + continue-on-error: false + - name: Setup pyenv + run: | + curl https://pyenv.run | bash + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv install 3.8:latest + pyenv install 3.9:latest + pyenv install 3.10:latest + pyenv install 3.11:latest + pyenv install 3.12:latest + pyenv install 3.13:latest + pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 -# # Verify installations -# echo "Installed versions:" -# pyenv versions -# - name: Verify pyenv installations -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# echo "Installed Python versions:" -# pyenv versions -# echo "" -# echo "Verifying all required Python versions are available:" -# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do -# if ! pyenv versions --bare | grep -q "^$version"; then -# echo "ERROR: Python $version is not installed!" -# exit 1 -# fi -# echo "✓ Python $version is installed" -# done -# echo "All Python versions verified successfully!" -# - name: Install dependencies for all Python versions -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do -# echo "Installing dependencies for Python $version" -# pyenv shell $version -# python -m pip install --upgrade pip -# python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel -# pyenv shell --unset -# done -# - name: Upgrade Rust toolchain -# run: | -# rustup toolchain install nightly-2025-07-07 -# rustup default nightly-2025-07-07 -# rustup component add rust-src -# rustc --version -# cargo --version -# - name: Install clang++ for Ubuntu -# run: | -# pwd -# uname -a -# wget https://apt.llvm.org/llvm.sh -# chmod +x llvm.sh -# sudo ./llvm.sh 19 -# which clang++-19 -# clang++-19 --version -# sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget -# # Install WebAssembly linker (wasm-ld) -# sudo apt-get install -y lld-19 -# # Create symlink for wasm-ld -# if ! command -v wasm-ld &> /dev/null; then -# sudo ln -sf /usr/bin/wasm-ld-19 /usr/bin/wasm-ld || true -# fi -# which wasm-ld || echo "wasm-ld not found in PATH" -# ccache -s -# - name: Update git -# run: | -# sudo add-apt-repository ppa:git-core/ppa -y -# sudo apt-get update -# sudo apt-get install -y git -# git --version -# - uses: actions/checkout@v3 -# with: -# fetch-depth: 0 -# - name: Update submodules -# run: | -# git submodule update --init --recursive --jobs 4 -# - name: ccache -# uses: hendrikmuhs/ccache-action@v1.2 -# with: -# key: ubuntu-24.04-aarch64 -# max-size: 5G -# append-timestamp: true -# - name: remove old clang and link clang-19 to clang -# run: | -# sudo rm -f /usr/bin/clang || true -# sudo ln -s /usr/bin/clang-19 /usr/bin/clang -# sudo rm -f /usr/bin/clang++ || true -# sudo ln -s /usr/bin/clang++-19 /usr/bin/clang++ -# which clang++ -# clang++ --version -# - name: Run chdb/build.sh -# timeout-minutes: 600 -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# source ~/.cargo/env -# pyenv shell 3.8 -# export CC=/usr/bin/clang -# export CXX=/usr/bin/clang++ -# bash ./chdb/build.sh -# pyenv shell 3.8 -# bash -x ./chdb/test_smoke.sh -# continue-on-error: false -# - name: Run chdb/build/build_static_lib.sh -# timeout-minutes: 600 -# run: | -# export CC=/usr/bin/clang -# export CXX=/usr/bin/clang++ -# export PATH="$HOME/.pyenv/bin:$PATH" -# source ~/.cargo/env -# eval "$(pyenv init -)" -# pyenv shell 3.8 -# bash ./chdb/build/build_static_lib.sh -# pyenv shell --unset -# continue-on-error: false -# - name: Scan chdb libraries with grype -# run: | -# echo "Scanning chdb libraries for vulnerabilities..." + # Verify installations + echo "Installed versions:" + pyenv versions + - name: Verify pyenv installations + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + echo "Installed Python versions:" + pyenv versions + echo "" + echo "Verifying all required Python versions are available:" + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + if ! pyenv versions --bare | grep -q "^$version"; then + echo "ERROR: Python $version is not installed!" + exit 1 + fi + echo "✓ Python $version is installed" + done + echo "All Python versions verified successfully!" + - name: Install dependencies for all Python versions + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + echo "Installing dependencies for Python $version" + pyenv shell $version + python -m pip install --upgrade pip + python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel + pyenv shell --unset + done + - name: Upgrade Rust toolchain + run: | + rustup toolchain install nightly-2025-07-07 + rustup default nightly-2025-07-07 + rustup component add rust-src + rustc --version + cargo --version + - name: Install clang++ for Ubuntu + run: | + pwd + uname -a + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 19 + which clang++-19 + clang++-19 --version + sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget + # Install WebAssembly linker (wasm-ld) + sudo apt-get install -y lld-19 + # Create symlink for wasm-ld + if ! command -v wasm-ld &> /dev/null; then + sudo ln -sf /usr/bin/wasm-ld-19 /usr/bin/wasm-ld || true + fi + which wasm-ld || echo "wasm-ld not found in PATH" + ccache -s + - name: Update git + run: | + sudo add-apt-repository ppa:git-core/ppa -y + sudo apt-get update + sudo apt-get install -y git + git --version + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Update submodules + run: | + git submodule update --init --recursive --jobs 4 + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ubuntu-24.04-aarch64 + max-size: 5G + append-timestamp: true + - name: remove old clang and link clang-19 to clang + run: | + sudo rm -f /usr/bin/clang || true + sudo ln -s /usr/bin/clang-19 /usr/bin/clang + sudo rm -f /usr/bin/clang++ || true + sudo ln -s /usr/bin/clang++-19 /usr/bin/clang++ + which clang++ + clang++ --version + - name: Run chdb/build.sh + timeout-minutes: 600 + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + source ~/.cargo/env + pyenv shell 3.8 + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + bash ./chdb/build.sh + pyenv shell 3.8 + bash -x ./chdb/test_smoke.sh + continue-on-error: false + - name: Run chdb/build/build_static_lib.sh + timeout-minutes: 600 + run: | + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + export PATH="$HOME/.pyenv/bin:$PATH" + source ~/.cargo/env + eval "$(pyenv init -)" + pyenv shell 3.8 + bash ./chdb/build/build_static_lib.sh + pyenv shell --unset + continue-on-error: false + - name: Scan chdb libraries with grype + run: | + echo "Scanning chdb libraries for vulnerabilities..." -# # Files to scan -# FILES_TO_SCAN="" -# [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" -# [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" -# FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" + # Files to scan + FILES_TO_SCAN="" + [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" + [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" + FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" -# SQLITE_VULNERABILITIES_FOUND=false + SQLITE_VULNERABILITIES_FOUND=false -# for file in $FILES_TO_SCAN; do -# if [ -f "$file" ]; then -# echo "=== Scanning $file ===" -# SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) -# echo "$SCAN_OUTPUT" + for file in $FILES_TO_SCAN; do + if [ -f "$file" ]; then + echo "=== Scanning $file ===" + SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) + echo "$SCAN_OUTPUT" -# if echo "$SCAN_OUTPUT" | grep -qi sqlite; then -# echo "❌ SQLite vulnerability found in $file" -# SQLITE_VULNERABILITIES_FOUND=true -# fi -# fi -# done + if echo "$SCAN_OUTPUT" | grep -qi sqlite; then + echo "❌ SQLite vulnerability found in $file" + SQLITE_VULNERABILITIES_FOUND=true + fi + fi + done -# if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then -# echo "❌ SQLite vulnerabilities detected in chdb libraries!" -# exit 1 -# else -# echo "✅ No SQLite vulnerabilities found in chdb libraries" -# fi -# continue-on-error: false -# - name: Run libchdb stub in examples dir -# run: | -# bash -x ./examples/runStub.sh -# - name: Check ccache statistics -# run: | -# ccache -s -# ls -lh chdb -# df -h -# - name: Build wheels -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# export CC=/usr/bin/clang -# export CXX=/usr/bin/clang++ -# pyenv shell 3.8 -# make wheel -# - name: Install patchelf from github -# run: | -# wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz -# tar -xvf patchelf.tar.gz -# sudo cp bin/patchelf /usr/bin/ -# sudo chmod +x /usr/bin/patchelf -# patchelf --version -# - name: Audit wheels -# run: | -# python3 -m pip install auditwheel -# auditwheel -v repair -w dist/ --plat manylinux_2_17_aarch64 dist/*.whl -# continue-on-error: false -# - name: Show files -# run: | -# # e.g: remove chdb-0.11.4-cp310-cp310-linux_aarch64.whl, keep chdb-0.11.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl -# sudo rm -f dist/*linux_aarch64.whl -# ls -lh dist -# shell: bash -# - name: Setup core dump collection -# run: | -# mkdir -p tmp/core -# echo "tmp/core/core.%p" | sudo tee /proc/sys/kernel/core_pattern -# ulimit -c unlimited -# - name: Test wheel on all Python versions -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do -# echo "Testing chdb on Python $version" -# pyenv shell $version -# python -m pip install dist/*.whl --force-reinstall -# python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" -# make test -# pyenv shell --unset -# done -# continue-on-error: false -# - name: Check and upload core files if present -# if: always() -# run: | -# if ls tmp/core/core.* >/dev/null 2>&1; then -# echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV -# tar -czvf core-files-linux-aarch64.tar.gz tmp/core/core.* -# echo "Core files tar created: core-files-linux-aarch64.tar.gz" -# ls -lh core-files-linux-aarch64.tar.gz -# else -# echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV -# echo "No core files found in tmp/core" -# fi -# continue-on-error: true -# - name: Upload core files if present -# if: always() && env.CORE_FILES_FOUND == 'true' -# uses: actions/upload-artifact@v4 -# with: -# name: core-files-linux-aarch64 -# path: core-files-linux-aarch64.tar.gz -# - name: Upload wheels to release -# if: startsWith(github.ref, 'refs/tags/v') -# run: | -# gh release upload ${{ github.ref_name }} dist/*.whl --clobber -# env: -# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} -# - name: Packege libchdb.so -# run: | -# cp programs/local/chdb.h chdb.h -# cp programs/local/chdb.hpp chdb.hpp -# tar -czvf linux-aarch64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp -# - name: Package libchdb.a -# run: | -# cp programs/local/chdb.h chdb.h -# cp programs/local/chdb.hpp chdb.hpp -# tar -czvf linux-aarch64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp -# - name: Upload libchdb.so to release -# if: startsWith(github.ref, 'refs/tags/v') -# run: | -# gh release upload ${{ github.ref_name }} linux-aarch64-libchdb.tar.gz --clobber -# env: -# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} -# - name: Upload libchdb.a to release -# if: startsWith(github.ref, 'refs/tags/v') -# run: | -# gh release upload ${{ github.ref_name }} linux-aarch64-libchdb-static.tar.gz --clobber -# env: -# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} -# - uses: actions/upload-artifact@v4 -# with: -# name: chdb-artifacts-linux-aarch64 -# path: | -# ./dist/*.whl -# ./linux-aarch64-libchdb.tar.gz -# ./linux-aarch64-libchdb-static.tar.gz -# overwrite: true -# - name: Upload pypi -# if: startsWith(github.ref, 'refs/tags/v') -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# pyenv shell 3.8 -# python -m twine upload dist/*.whl -# env: -# TWINE_USERNAME: __token__ -# TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then + echo "❌ SQLite vulnerabilities detected in chdb libraries!" + exit 1 + else + echo "✅ No SQLite vulnerabilities found in chdb libraries" + fi + continue-on-error: false + - name: Run libchdb stub in examples dir + run: | + bash -x ./examples/runStub.sh + - name: Check ccache statistics + run: | + ccache -s + ls -lh chdb + df -h + - name: Build wheels + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + pyenv shell 3.8 + make wheel + - name: Install patchelf from github + run: | + wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz + tar -xvf patchelf.tar.gz + sudo cp bin/patchelf /usr/bin/ + sudo chmod +x /usr/bin/patchelf + patchelf --version + - name: Audit wheels + run: | + python3 -m pip install auditwheel + auditwheel -v repair -w dist/ --plat manylinux_2_17_aarch64 dist/*.whl + continue-on-error: false + - name: Show files + run: | + # e.g: remove chdb-0.11.4-cp310-cp310-linux_aarch64.whl, keep chdb-0.11.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl + sudo rm -f dist/*linux_aarch64.whl + ls -lh dist + shell: bash + - name: Setup core dump collection + run: | + mkdir -p tmp/core + echo "tmp/core/core.%p" | sudo tee /proc/sys/kernel/core_pattern + ulimit -c unlimited + - name: Test wheel on all Python versions + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + echo "Testing chdb on Python $version" + pyenv shell $version + python -m pip install dist/*.whl --force-reinstall + python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" + make test + pyenv shell --unset + done + continue-on-error: false + - name: Check and upload core files if present + if: always() + run: | + if ls tmp/core/core.* >/dev/null 2>&1; then + echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV + tar -czvf core-files-linux-aarch64.tar.gz tmp/core/core.* + echo "Core files tar created: core-files-linux-aarch64.tar.gz" + ls -lh core-files-linux-aarch64.tar.gz + else + echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV + echo "No core files found in tmp/core" + fi + continue-on-error: true + - name: Upload core files if present + if: always() && env.CORE_FILES_FOUND == 'true' + uses: actions/upload-artifact@v4 + with: + name: core-files-linux-aarch64 + path: core-files-linux-aarch64.tar.gz + - name: Upload wheels to release + if: startsWith(github.ref, 'refs/tags/v') + run: | + gh release upload ${{ github.ref_name }} dist/*.whl --clobber + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + - name: Packege libchdb.so + run: | + cp programs/local/chdb.h chdb.h + cp programs/local/chdb.hpp chdb.hpp + tar -czvf linux-aarch64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp + - name: Package libchdb.a + run: | + cp programs/local/chdb.h chdb.h + cp programs/local/chdb.hpp chdb.hpp + tar -czvf linux-aarch64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp + - name: Upload libchdb.so to release + if: startsWith(github.ref, 'refs/tags/v') + run: | + gh release upload ${{ github.ref_name }} linux-aarch64-libchdb.tar.gz --clobber + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + - name: Upload libchdb.a to release + if: startsWith(github.ref, 'refs/tags/v') + run: | + gh release upload ${{ github.ref_name }} linux-aarch64-libchdb-static.tar.gz --clobber + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + - uses: actions/upload-artifact@v4 + with: + name: chdb-artifacts-linux-aarch64 + path: | + ./dist/*.whl + ./linux-aarch64-libchdb.tar.gz + ./linux-aarch64-libchdb-static.tar.gz + overwrite: true + - name: Upload pypi + if: startsWith(github.ref, 'refs/tags/v') + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv shell 3.8 + python -m twine upload dist/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/build_linux_x86_wheels.yml b/.github/workflows/build_linux_x86_wheels.yml index 29a71bbb668..5a35668ead9 100644 --- a/.github/workflows/build_linux_x86_wheels.yml +++ b/.github/workflows/build_linux_x86_wheels.yml @@ -1,332 +1,332 @@ -# name: Build Linux X86 +name: Build Linux X86 -# on: -# workflow_dispatch: -# inputs: -# TAG_NAME: -# description: 'Release Version Tag' -# required: true -# release: -# types: [created] -# push: -# branches: -# - main -# paths-ignore: -# - '**/*.md' -# pull_request: -# branches: -# - main -# paths-ignore: -# - '**/*.md' +on: + workflow_dispatch: + inputs: + TAG_NAME: + description: 'Release Version Tag' + required: true + release: + types: [created] + push: + branches: + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - main + paths-ignore: + - '**/*.md' -# jobs: -# build_universal_wheel: -# name: Build Universal Wheel (Linux x86_64) -# runs-on: gh-64c -# steps: -# - name: Install Python build dependencies -# run: | -# sudo apt-get update -# sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \ -# libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ -# libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \ -# libffi-dev liblzma-dev -# - name: Scan SQLite vulnerabilities with grype -# run: | -# # Install grype and required tools -# curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin -# sudo apt-get update && sudo apt-get install -y jq lsb-release +jobs: + build_universal_wheel: + name: Build Universal Wheel (Linux x86_64) + runs-on: gh-64c + steps: + - name: Install Python build dependencies + run: | + sudo apt-get update + sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \ + libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ + libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \ + libffi-dev liblzma-dev + - name: Scan SQLite vulnerabilities with grype + run: | + # Install grype and required tools + curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin + sudo apt-get update && sudo apt-get install -y jq lsb-release -# # Detect OS distribution info -# DISTRO_ID=$(lsb_release -si | tr '[:upper:]' '[:lower:]') -# DISTRO_VERSION=$(lsb_release -sr) -# echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION" + # Detect OS distribution info + DISTRO_ID=$(lsb_release -si | tr '[:upper:]' '[:lower:]') + DISTRO_VERSION=$(lsb_release -sr) + echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION" -# # Update grype vulnerability database -# grype db update + # Update grype vulnerability database + grype db update -# # Check SQLite vulnerabilities in installed packages -# echo "Scanning SQLite packages for vulnerabilities..." -# GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true) -# echo "Raw grype output:" -# echo "$GRYPE_RAW_OUTPUT" + # Check SQLite vulnerabilities in installed packages + echo "Scanning SQLite packages for vulnerabilities..." + GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true) + echo "Raw grype output:" + echo "$GRYPE_RAW_OUTPUT" -# SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) + SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) -# if [ -n "$SQLITE_SCAN_OUTPUT" ]; then -# echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." -# echo "SQLite vulnerability details:" -# echo "$SQLITE_SCAN_OUTPUT" -# exit 1 -# else -# echo "✅ No SQLite vulnerabilities found" -# fi -# continue-on-error: false -# - name: Setup pyenv -# run: | -# curl https://pyenv.run | bash -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# pyenv install 3.8:latest -# pyenv install 3.9:latest -# pyenv install 3.10:latest -# pyenv install 3.11:latest -# pyenv install 3.12:latest -# pyenv install 3.13:latest -# pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 + if [ -n "$SQLITE_SCAN_OUTPUT" ]; then + echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." + echo "SQLite vulnerability details:" + echo "$SQLITE_SCAN_OUTPUT" + exit 1 + else + echo "✅ No SQLite vulnerabilities found" + fi + continue-on-error: false + - name: Setup pyenv + run: | + curl https://pyenv.run | bash + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv install 3.8:latest + pyenv install 3.9:latest + pyenv install 3.10:latest + pyenv install 3.11:latest + pyenv install 3.12:latest + pyenv install 3.13:latest + pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 -# # Verify installations -# echo "Installed versions:" -# pyenv versions -# - name: Verify pyenv installations -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# echo "Installed Python versions:" -# pyenv versions -# echo "" -# echo "Verifying all required Python versions are available:" -# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do -# if ! pyenv versions --bare | grep -q "^$version"; then -# echo "ERROR: Python $version is not installed!" -# exit 1 -# fi -# echo "✓ Python $version is installed" -# done -# echo "All Python versions verified successfully!" -# - name: Install dependencies for all Python versions -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do -# echo "Installing dependencies for Python $version" -# pyenv shell $version -# python -m pip install --upgrade pip -# python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel -# pyenv shell --unset -# done -# - name: Upgrade Rust toolchain -# run: | -# rustup toolchain install nightly-2025-07-07 -# rustup default nightly-2025-07-07 -# rustup component add rust-src -# rustc --version -# cargo --version -# - name: Install clang++ for Ubuntu -# run: | -# pwd -# uname -a -# wget https://apt.llvm.org/llvm.sh -# chmod +x llvm.sh -# sudo ./llvm.sh 19 -# which clang++-19 -# clang++-19 --version -# sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget -# # Install WebAssembly linker (wasm-ld) -# sudo apt-get install -y lld-19 -# # Create symlink for wasm-ld -# if ! command -v wasm-ld &> /dev/null; then -# sudo ln -sf /usr/bin/wasm-ld-19 /usr/bin/wasm-ld || true -# fi -# which wasm-ld || echo "wasm-ld not found in PATH" -# ccache -s -# - name: Update git -# run: | -# sudo add-apt-repository ppa:git-core/ppa -y -# sudo apt-get update -# sudo apt-get install -y git -# git --version -# - uses: actions/checkout@v3 -# with: -# fetch-depth: 0 -# - name: Update submodules -# run: | -# git submodule update --init --recursive --jobs 4 -# - name: ccache -# uses: hendrikmuhs/ccache-action@v1.2 -# with: -# key: ubuntu-22.04-x86_64 -# max-size: 5G -# append-timestamp: true -# - name: remove old clang and link clang-19 to clang -# run: | -# sudo rm -f /usr/bin/clang || true -# sudo ln -s /usr/bin/clang-19 /usr/bin/clang -# sudo rm -f /usr/bin/clang++ || true -# sudo ln -s /usr/bin/clang++-19 /usr/bin/clang++ -# which clang++ -# clang++ --version -# - name: Run chdb/build.sh -# timeout-minutes: 600 -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# source ~/.cargo/env -# pyenv shell 3.8 -# export CC=/usr/bin/clang -# export CXX=/usr/bin/clang++ -# bash ./chdb/build.sh -# pyenv shell 3.8 -# bash -x ./chdb/test_smoke.sh -# continue-on-error: false -# - name: Run chdb/build/build_static_lib.sh -# timeout-minutes: 600 -# run: | -# export CC=/usr/bin/clang -# export CXX=/usr/bin/clang++ -# export PATH="$HOME/.pyenv/bin:$PATH" -# source ~/.cargo/env -# eval "$(pyenv init -)" -# pyenv shell 3.8 -# bash ./chdb/build/build_static_lib.sh -# pyenv shell --unset -# continue-on-error: false -# - name: Scan chdb libraries with grype -# run: | -# echo "Scanning chdb libraries for vulnerabilities..." + # Verify installations + echo "Installed versions:" + pyenv versions + - name: Verify pyenv installations + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + echo "Installed Python versions:" + pyenv versions + echo "" + echo "Verifying all required Python versions are available:" + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + if ! pyenv versions --bare | grep -q "^$version"; then + echo "ERROR: Python $version is not installed!" + exit 1 + fi + echo "✓ Python $version is installed" + done + echo "All Python versions verified successfully!" + - name: Install dependencies for all Python versions + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + echo "Installing dependencies for Python $version" + pyenv shell $version + python -m pip install --upgrade pip + python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel + pyenv shell --unset + done + - name: Upgrade Rust toolchain + run: | + rustup toolchain install nightly-2025-07-07 + rustup default nightly-2025-07-07 + rustup component add rust-src + rustc --version + cargo --version + - name: Install clang++ for Ubuntu + run: | + pwd + uname -a + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 19 + which clang++-19 + clang++-19 --version + sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget + # Install WebAssembly linker (wasm-ld) + sudo apt-get install -y lld-19 + # Create symlink for wasm-ld + if ! command -v wasm-ld &> /dev/null; then + sudo ln -sf /usr/bin/wasm-ld-19 /usr/bin/wasm-ld || true + fi + which wasm-ld || echo "wasm-ld not found in PATH" + ccache -s + - name: Update git + run: | + sudo add-apt-repository ppa:git-core/ppa -y + sudo apt-get update + sudo apt-get install -y git + git --version + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Update submodules + run: | + git submodule update --init --recursive --jobs 4 + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ubuntu-22.04-x86_64 + max-size: 5G + append-timestamp: true + - name: remove old clang and link clang-19 to clang + run: | + sudo rm -f /usr/bin/clang || true + sudo ln -s /usr/bin/clang-19 /usr/bin/clang + sudo rm -f /usr/bin/clang++ || true + sudo ln -s /usr/bin/clang++-19 /usr/bin/clang++ + which clang++ + clang++ --version + - name: Run chdb/build.sh + timeout-minutes: 600 + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + source ~/.cargo/env + pyenv shell 3.8 + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + bash ./chdb/build.sh + pyenv shell 3.8 + bash -x ./chdb/test_smoke.sh + continue-on-error: false + - name: Run chdb/build/build_static_lib.sh + timeout-minutes: 600 + run: | + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + export PATH="$HOME/.pyenv/bin:$PATH" + source ~/.cargo/env + eval "$(pyenv init -)" + pyenv shell 3.8 + bash ./chdb/build/build_static_lib.sh + pyenv shell --unset + continue-on-error: false + - name: Scan chdb libraries with grype + run: | + echo "Scanning chdb libraries for vulnerabilities..." -# # Files to scan -# FILES_TO_SCAN="" -# [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" -# [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" -# FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" + # Files to scan + FILES_TO_SCAN="" + [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" + [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" + FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" -# SQLITE_VULNERABILITIES_FOUND=false + SQLITE_VULNERABILITIES_FOUND=false -# for file in $FILES_TO_SCAN; do -# if [ -f "$file" ]; then -# echo "=== Scanning $file ===" -# SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) -# echo "$SCAN_OUTPUT" + for file in $FILES_TO_SCAN; do + if [ -f "$file" ]; then + echo "=== Scanning $file ===" + SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) + echo "$SCAN_OUTPUT" -# if echo "$SCAN_OUTPUT" | grep -qi sqlite; then -# echo "❌ SQLite vulnerability found in $file" -# SQLITE_VULNERABILITIES_FOUND=true -# fi -# fi -# done + if echo "$SCAN_OUTPUT" | grep -qi sqlite; then + echo "❌ SQLite vulnerability found in $file" + SQLITE_VULNERABILITIES_FOUND=true + fi + fi + done -# if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then -# echo "❌ SQLite vulnerabilities detected in chdb libraries!" -# exit 1 -# else -# echo "✅ No SQLite vulnerabilities found in chdb libraries" -# fi -# continue-on-error: false -# - name: Run libchdb stub in examples dir -# run: | -# bash -x ./examples/runStub.sh -# - name: Check ccache statistics -# run: | -# ccache -s -# ls -lh chdb -# df -h -# - name: Build wheels -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# export CC=/usr/bin/clang -# export CXX=/usr/bin/clang++ -# pyenv shell 3.8 -# make wheel -# - name: Install patchelf from github -# run: | -# wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz -O patchelf.tar.gz -# tar -xvf patchelf.tar.gz -# sudo cp bin/patchelf /usr/bin/ -# sudo chmod +x /usr/bin/patchelf -# patchelf --version -# - name: Audit wheels -# run: | -# python3 -m pip install auditwheel -# auditwheel -v repair -w dist/ --plat manylinux2014_x86_64 dist/*.whl -# continue-on-error: false -# - name: Show files -# run: | -# sudo rm -f dist/*-linux_x86_64.whl -# ls -lh dist -# shell: bash -# - name: Setup core dump collection -# run: | -# mkdir -p tmp/core -# echo "tmp/core/core.%p" | sudo tee /proc/sys/kernel/core_pattern -# ulimit -c unlimited -# - name: Test wheel on all Python versions -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do -# echo "Testing chdb on Python $version" -# pyenv shell $version -# python -m pip install dist/*.whl --force-reinstall -# python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" -# make test -# pyenv shell --unset -# done -# continue-on-error: false -# - name: Check and upload core files if present -# if: always() -# run: | -# if ls tmp/core/core.* >/dev/null 2>&1; then -# echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV -# tar -czvf core-files-linux-x86_64.tar.gz tmp/core/core.* -# echo "Core files tar created: core-files-linux-x86_64.tar.gz" -# ls -lh core-files-linux-x86_64.tar.gz -# else -# echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV -# echo "No core files found in tmp/core" -# fi -# continue-on-error: true -# - name: Upload core files artifact -# if: always() && env.CORE_FILES_FOUND == 'true' -# uses: actions/upload-artifact@v4 -# with: -# name: core-files-linux-x86_64 -# path: core-files-linux-x86_64.tar.gz -# - name: Upload wheels to release -# if: startsWith(github.ref, 'refs/tags/v') -# run: | -# gh release upload ${{ github.ref_name }} dist/*.whl --clobber -# env: -# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} -# - name: Packege libchdb.so -# run: | -# cp programs/local/chdb.h chdb.h -# cp programs/local/chdb.hpp chdb.hpp -# tar -czvf linux-x86_64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp -# - name: Package libchdb.a -# run: | -# cp programs/local/chdb.h chdb.h -# cp programs/local/chdb.hpp chdb.hpp -# tar -czvf linux-x86_64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp -# - name: Upload libchdb.so to release -# if: startsWith(github.ref, 'refs/tags/v') -# run: | -# gh release upload ${{ github.ref_name }} linux-x86_64-libchdb.tar.gz --clobber -# env: -# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} -# - name: Upload libchdb.a to release -# if: startsWith(github.ref, 'refs/tags/v') -# run: | -# gh release upload ${{ github.ref_name }} linux-x86_64-libchdb-static.tar.gz --clobber -# env: -# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} -# - uses: actions/upload-artifact@v4 -# with: -# name: chdb-artifacts-linux-x86_64 -# path: | -# ./dist/*.whl -# ./linux-x86_64-libchdb.tar.gz -# ./linux-x86_64-libchdb-static.tar.gz -# overwrite: true -# - name: Upload pypi -# if: startsWith(github.ref, 'refs/tags/v') -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# pyenv shell 3.8 -# python -m twine upload dist/*.whl -# env: -# TWINE_USERNAME: __token__ -# TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then + echo "❌ SQLite vulnerabilities detected in chdb libraries!" + exit 1 + else + echo "✅ No SQLite vulnerabilities found in chdb libraries" + fi + continue-on-error: false + - name: Run libchdb stub in examples dir + run: | + bash -x ./examples/runStub.sh + - name: Check ccache statistics + run: | + ccache -s + ls -lh chdb + df -h + - name: Build wheels + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + pyenv shell 3.8 + make wheel + - name: Install patchelf from github + run: | + wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz -O patchelf.tar.gz + tar -xvf patchelf.tar.gz + sudo cp bin/patchelf /usr/bin/ + sudo chmod +x /usr/bin/patchelf + patchelf --version + - name: Audit wheels + run: | + python3 -m pip install auditwheel + auditwheel -v repair -w dist/ --plat manylinux2014_x86_64 dist/*.whl + continue-on-error: false + - name: Show files + run: | + sudo rm -f dist/*-linux_x86_64.whl + ls -lh dist + shell: bash + - name: Setup core dump collection + run: | + mkdir -p tmp/core + echo "tmp/core/core.%p" | sudo tee /proc/sys/kernel/core_pattern + ulimit -c unlimited + - name: Test wheel on all Python versions + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + echo "Testing chdb on Python $version" + pyenv shell $version + python -m pip install dist/*.whl --force-reinstall + python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" + make test + pyenv shell --unset + done + continue-on-error: false + - name: Check and upload core files if present + if: always() + run: | + if ls tmp/core/core.* >/dev/null 2>&1; then + echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV + tar -czvf core-files-linux-x86_64.tar.gz tmp/core/core.* + echo "Core files tar created: core-files-linux-x86_64.tar.gz" + ls -lh core-files-linux-x86_64.tar.gz + else + echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV + echo "No core files found in tmp/core" + fi + continue-on-error: true + - name: Upload core files artifact + if: always() && env.CORE_FILES_FOUND == 'true' + uses: actions/upload-artifact@v4 + with: + name: core-files-linux-x86_64 + path: core-files-linux-x86_64.tar.gz + - name: Upload wheels to release + if: startsWith(github.ref, 'refs/tags/v') + run: | + gh release upload ${{ github.ref_name }} dist/*.whl --clobber + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + - name: Packege libchdb.so + run: | + cp programs/local/chdb.h chdb.h + cp programs/local/chdb.hpp chdb.hpp + tar -czvf linux-x86_64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp + - name: Package libchdb.a + run: | + cp programs/local/chdb.h chdb.h + cp programs/local/chdb.hpp chdb.hpp + tar -czvf linux-x86_64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp + - name: Upload libchdb.so to release + if: startsWith(github.ref, 'refs/tags/v') + run: | + gh release upload ${{ github.ref_name }} linux-x86_64-libchdb.tar.gz --clobber + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + - name: Upload libchdb.a to release + if: startsWith(github.ref, 'refs/tags/v') + run: | + gh release upload ${{ github.ref_name }} linux-x86_64-libchdb-static.tar.gz --clobber + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + - uses: actions/upload-artifact@v4 + with: + name: chdb-artifacts-linux-x86_64 + path: | + ./dist/*.whl + ./linux-x86_64-libchdb.tar.gz + ./linux-x86_64-libchdb-static.tar.gz + overwrite: true + - name: Upload pypi + if: startsWith(github.ref, 'refs/tags/v') + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv shell 3.8 + python -m twine upload dist/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/build_macos_arm64_wheels.yml b/.github/workflows/build_macos_arm64_wheels.yml index 255867f16b6..61c895f7601 100644 --- a/.github/workflows/build_macos_arm64_wheels.yml +++ b/.github/workflows/build_macos_arm64_wheels.yml @@ -1,276 +1,276 @@ -# name: Build macOS arm64 +name: Build macOS arm64 -# on: -# workflow_dispatch: -# inputs: -# TAG_NAME: -# description: 'Release Version Tag' -# required: true -# release: -# types: [created] -# push: -# branches: -# - main -# paths-ignore: -# - '**/*.md' -# pull_request: -# branches: -# - main -# paths-ignore: -# - '**/*.md' +on: + workflow_dispatch: + inputs: + TAG_NAME: + description: 'Release Version Tag' + required: true + release: + types: [created] + push: + branches: + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - main + paths-ignore: + - '**/*.md' -# jobs: -# build_universal_wheel: -# name: Build Universal Wheel (macOS ARM64) -# runs-on: macos-13-xlarge -# steps: -# - name: Setup pyenv -# run: | -# curl https://pyenv.run | bash -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" +jobs: + build_universal_wheel: + name: Build Universal Wheel (macOS ARM64) + runs-on: macos-13-xlarge + steps: + - name: Setup pyenv + run: | + curl https://pyenv.run | bash + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" -# pyenv install 3.8:latest -# pyenv install 3.9:latest -# pyenv install 3.10:latest -# pyenv install 3.11:latest -# pyenv install 3.12:latest -# pyenv install 3.13:latest -# pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 + pyenv install 3.8:latest + pyenv install 3.9:latest + pyenv install 3.10:latest + pyenv install 3.11:latest + pyenv install 3.12:latest + pyenv install 3.13:latest + pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 -# echo "Installed versions:" -# pyenv versions -# - name: Verify pyenv installations -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# echo "Installed Python versions:" -# pyenv versions -# echo "" -# echo "Verifying all required Python versions are available:" -# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do -# if ! pyenv versions --bare | grep -q "^$version"; then -# echo "ERROR: Python $version is not installed!" -# exit 1 -# fi -# echo "✓ Python $version is installed" -# done -# echo "All Python versions verified successfully!" -# - name: Install dependencies for all Python versions -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do -# echo "Installing dependencies for Python $version" -# pyenv shell $version -# python -m pip install --upgrade pip -# python -m pip install setuptools wheel tox pandas pyarrow twine psutil deltalake wheel>=0.40.0 -# pyenv shell --unset -# done -# - name: Remove /usr/local/bin/python3 -# run: | -# sudo rm -f /usr/local/bin/python3 -# - name: Install clang++ for macOS -# run: | -# pwd -# uname -a -# export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 -# brew update -# brew install ca-certificates lz4 mpdecimal readline sqlite xz z3 zstd -# brew install openssl@3 || echo "OpenSSL install failed, continuing..." -# brew install --ignore-dependencies llvm@19 -# brew install git ninja libtool gettext binutils grep findutils nasm lld@19 libiconv -# brew install go -# cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm -# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:$PATH -# which clang++ -# clang++ --version -# which wasm-ld || echo "wasm-ld not found in PATH" -# which go -# go version -# ccache -s | echo "ccache not available yet" -# - name: Upgrade Rust toolchain -# run: | -# rustup toolchain install nightly-2025-07-07 -# rustup default nightly-2025-07-07 -# rustup component add rust-src -# rustc --version -# cargo --version -# - uses: actions/checkout@v3 -# with: -# fetch-depth: 0 -# - name: Update submodules -# run: | -# git submodule update --init --recursive --jobs 4 -# - name: Run chdb/build.sh -# timeout-minutes: 600 -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# source ~/.cargo/env -# pyenv shell 3.8 -# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin -# export CC=$(brew --prefix llvm@19)/bin/clang -# export CXX=$(brew --prefix llvm@19)/bin/clang++ -# bash gen_manifest.sh -# bash ./chdb/build.sh -# pyenv shell 3.8 -# bash -x ./chdb/test_smoke.sh -# - name: Run chdb/build/build_static_lib.sh -# timeout-minutes: 600 -# run: | -# export PATH=$HOME/.pyenv/bin:$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin -# export CC=$(brew --prefix llvm@19)/bin/clang -# export CXX=$(brew --prefix llvm@19)/bin/clang++ -# source ~/.cargo/env -# eval "$(pyenv init -)" -# pyenv shell 3.8 -# bash ./chdb/build/build_static_lib.sh -# pyenv shell --unset -# continue-on-error: false -# - name: Debug libchdb -# run: | -# ls -lh -# llvm-nm libchdb.so | grep query_stable || true -# echo "Global Symbol in libchdb.so:" -# llvm-nm -g libchdb.so || true -# echo "Global Symbol in libclickhouse-local-chdb.a:" -# llvm-nm -g buildlib/programs/local/libclickhouse-local-chdb.a || true -# echo "Global Symbol in libclickhouse-local-lib.a:" -# llvm-nm -g buildlib/programs/local/libclickhouse-local-lib.a || true -# echo "pychdb_cmd.sh:" -# cat buildlib/pychdb_cmd.sh -# echo "libchdb_cmd.sh:" -# cat buildlib/libchdb_cmd.sh -# - name: Scan chdb libraries with grype -# run: | -# echo "Scanning chdb libraries for vulnerabilities..." -# # Files to scan -# FILES_TO_SCAN="" -# [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" -# [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" -# FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)" -# SQLITE_VULNERABILITIES_FOUND=false -# for file in $FILES_TO_SCAN; do -# if [ -f "$file" ]; then -# echo "=== Scanning $file ===" -# SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) -# echo "$SCAN_OUTPUT" -# if echo "$SCAN_OUTPUT" | grep -qi sqlite; then -# echo "❌ SQLite vulnerability found in $file" -# SQLITE_VULNERABILITIES_FOUND=true -# fi -# fi -# done -# if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then -# echo "❌ SQLite vulnerabilities detected in chdb libraries!" -# exit 1 -# else -# echo "✅ No SQLite vulnerabilities found in chdb libraries" -# fi -# continue-on-error: false -# - name: Run libchdb stub in examples dir -# run: | -# bash -x ./examples/runStub.sh -# - name: Build wheels -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin -# export CC=$(brew --prefix llvm@19)/bin/clang -# export CXX=$(brew --prefix llvm@19)/bin/clang++ -# pyenv shell 3.8 -# make wheel -# - name: Fix wheel platform tag -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# pyenv shell 3.8 -# python -m wheel tags --platform-tag=macosx_11_0_arm64 --remove dist/*.whl -# - name: Setup core dump -# run: | -# mkdir -p tmp/core -# sudo sysctl kern.corefile=$PWD/tmp/core/core.%P -# sudo sysctl kern.coredump=1 -# ulimit -c unlimited -# - name: Test wheel on all Python versions -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do -# echo "Testing chdb on Python $version" -# pyenv shell $version -# python -m pip install dist/*.whl --force-reinstall -# python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" -# make test -# pyenv shell --unset -# done -# continue-on-error: false -# - name: Check and upload core files if present -# if: always() -# run: | -# if ls tmp/core/core.* >/dev/null 2>&1; then -# echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV -# tar -czvf core-files-macos-arm64.tar.gz tmp/core/core.* -# echo "Core files tar created: core-files-macos-arm64.tar.gz" -# ls -lh core-files-macos-arm64.tar.gz -# else -# echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV -# echo "No core files found in tmp/core" -# fi -# continue-on-error: true -# - name: Upload core files artifact -# if: always() && env.CORE_FILES_FOUND == 'true' -# uses: actions/upload-artifact@v4 -# with: -# name: core-files-macos-arm64 -# path: core-files-macos-arm64.tar.gz -# - name: Show files -# run: ls -lh dist -# shell: bash -# - name: Upload wheels to release -# if: startsWith(github.ref, 'refs/tags/v') -# run: | -# gh release upload ${{ github.ref_name }} dist/*.whl --clobber -# env: -# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} -# - name: Packege libchdb.so -# run: | -# cp programs/local/chdb.h chdb.h -# cp programs/local/chdb.hpp chdb.hpp -# tar -czvf macos-arm64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp -# - name: Package libchdb.a -# run: | -# cp programs/local/chdb.h chdb.h -# cp programs/local/chdb.hpp chdb.hpp -# tar -czvf macos-arm64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp -# - name: Upload libchdb.so to release -# if: startsWith(github.ref, 'refs/tags/v') -# run: | -# gh release upload ${{ github.ref_name }} macos-arm64-libchdb.tar.gz --clobber -# env: -# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} -# - name: Upload libchdb.a to release -# if: startsWith(github.ref, 'refs/tags/v') -# run: | -# gh release upload ${{ github.ref_name }} macos-arm64-libchdb-static.tar.gz --clobber -# env: -# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} -# - uses: actions/upload-artifact@v4 -# with: -# name: chdb-artifacts-macos-arm64 -# path: | -# ./dist/*.whl -# ./macos-arm64-libchdb.tar.gz -# ./macos-arm64-libchdb-static.tar.gz -# overwrite: true -# - name: Upload pypi -# if: startsWith(github.ref, 'refs/tags/v') -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# pyenv shell 3.8 -# python -m twine upload dist/*.whl -# env: -# TWINE_USERNAME: __token__ -# TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + echo "Installed versions:" + pyenv versions + - name: Verify pyenv installations + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + echo "Installed Python versions:" + pyenv versions + echo "" + echo "Verifying all required Python versions are available:" + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + if ! pyenv versions --bare | grep -q "^$version"; then + echo "ERROR: Python $version is not installed!" + exit 1 + fi + echo "✓ Python $version is installed" + done + echo "All Python versions verified successfully!" + - name: Install dependencies for all Python versions + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + echo "Installing dependencies for Python $version" + pyenv shell $version + python -m pip install --upgrade pip + python -m pip install setuptools wheel tox pandas pyarrow twine psutil deltalake wheel>=0.40.0 + pyenv shell --unset + done + - name: Remove /usr/local/bin/python3 + run: | + sudo rm -f /usr/local/bin/python3 + - name: Install clang++ for macOS + run: | + pwd + uname -a + export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 + brew update + brew install ca-certificates lz4 mpdecimal readline sqlite xz z3 zstd + brew install openssl@3 || echo "OpenSSL install failed, continuing..." + brew install --ignore-dependencies llvm@19 + brew install git ninja libtool gettext binutils grep findutils nasm lld@19 libiconv + brew install go + cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm + export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:$PATH + which clang++ + clang++ --version + which wasm-ld || echo "wasm-ld not found in PATH" + which go + go version + ccache -s | echo "ccache not available yet" + - name: Upgrade Rust toolchain + run: | + rustup toolchain install nightly-2025-07-07 + rustup default nightly-2025-07-07 + rustup component add rust-src + rustc --version + cargo --version + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Update submodules + run: | + git submodule update --init --recursive --jobs 4 + - name: Run chdb/build.sh + timeout-minutes: 600 + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + source ~/.cargo/env + pyenv shell 3.8 + export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin + export CC=$(brew --prefix llvm@19)/bin/clang + export CXX=$(brew --prefix llvm@19)/bin/clang++ + bash gen_manifest.sh + bash ./chdb/build.sh + pyenv shell 3.8 + bash -x ./chdb/test_smoke.sh + - name: Run chdb/build/build_static_lib.sh + timeout-minutes: 600 + run: | + export PATH=$HOME/.pyenv/bin:$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin + export CC=$(brew --prefix llvm@19)/bin/clang + export CXX=$(brew --prefix llvm@19)/bin/clang++ + source ~/.cargo/env + eval "$(pyenv init -)" + pyenv shell 3.8 + bash ./chdb/build/build_static_lib.sh + pyenv shell --unset + continue-on-error: false + - name: Debug libchdb + run: | + ls -lh + llvm-nm libchdb.so | grep query_stable || true + echo "Global Symbol in libchdb.so:" + llvm-nm -g libchdb.so || true + echo "Global Symbol in libclickhouse-local-chdb.a:" + llvm-nm -g buildlib/programs/local/libclickhouse-local-chdb.a || true + echo "Global Symbol in libclickhouse-local-lib.a:" + llvm-nm -g buildlib/programs/local/libclickhouse-local-lib.a || true + echo "pychdb_cmd.sh:" + cat buildlib/pychdb_cmd.sh + echo "libchdb_cmd.sh:" + cat buildlib/libchdb_cmd.sh + - name: Scan chdb libraries with grype + run: | + echo "Scanning chdb libraries for vulnerabilities..." + # Files to scan + FILES_TO_SCAN="" + [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" + [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" + FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)" + SQLITE_VULNERABILITIES_FOUND=false + for file in $FILES_TO_SCAN; do + if [ -f "$file" ]; then + echo "=== Scanning $file ===" + SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) + echo "$SCAN_OUTPUT" + if echo "$SCAN_OUTPUT" | grep -qi sqlite; then + echo "❌ SQLite vulnerability found in $file" + SQLITE_VULNERABILITIES_FOUND=true + fi + fi + done + if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then + echo "❌ SQLite vulnerabilities detected in chdb libraries!" + exit 1 + else + echo "✅ No SQLite vulnerabilities found in chdb libraries" + fi + continue-on-error: false + - name: Run libchdb stub in examples dir + run: | + bash -x ./examples/runStub.sh + - name: Build wheels + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin + export CC=$(brew --prefix llvm@19)/bin/clang + export CXX=$(brew --prefix llvm@19)/bin/clang++ + pyenv shell 3.8 + make wheel + - name: Fix wheel platform tag + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv shell 3.8 + python -m wheel tags --platform-tag=macosx_11_0_arm64 --remove dist/*.whl + - name: Setup core dump + run: | + mkdir -p tmp/core + sudo sysctl kern.corefile=$PWD/tmp/core/core.%P + sudo sysctl kern.coredump=1 + ulimit -c unlimited + - name: Test wheel on all Python versions + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + echo "Testing chdb on Python $version" + pyenv shell $version + python -m pip install dist/*.whl --force-reinstall + python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" + make test + pyenv shell --unset + done + continue-on-error: false + - name: Check and upload core files if present + if: always() + run: | + if ls tmp/core/core.* >/dev/null 2>&1; then + echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV + tar -czvf core-files-macos-arm64.tar.gz tmp/core/core.* + echo "Core files tar created: core-files-macos-arm64.tar.gz" + ls -lh core-files-macos-arm64.tar.gz + else + echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV + echo "No core files found in tmp/core" + fi + continue-on-error: true + - name: Upload core files artifact + if: always() && env.CORE_FILES_FOUND == 'true' + uses: actions/upload-artifact@v4 + with: + name: core-files-macos-arm64 + path: core-files-macos-arm64.tar.gz + - name: Show files + run: ls -lh dist + shell: bash + - name: Upload wheels to release + if: startsWith(github.ref, 'refs/tags/v') + run: | + gh release upload ${{ github.ref_name }} dist/*.whl --clobber + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + - name: Packege libchdb.so + run: | + cp programs/local/chdb.h chdb.h + cp programs/local/chdb.hpp chdb.hpp + tar -czvf macos-arm64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp + - name: Package libchdb.a + run: | + cp programs/local/chdb.h chdb.h + cp programs/local/chdb.hpp chdb.hpp + tar -czvf macos-arm64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp + - name: Upload libchdb.so to release + if: startsWith(github.ref, 'refs/tags/v') + run: | + gh release upload ${{ github.ref_name }} macos-arm64-libchdb.tar.gz --clobber + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + - name: Upload libchdb.a to release + if: startsWith(github.ref, 'refs/tags/v') + run: | + gh release upload ${{ github.ref_name }} macos-arm64-libchdb-static.tar.gz --clobber + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + - uses: actions/upload-artifact@v4 + with: + name: chdb-artifacts-macos-arm64 + path: | + ./dist/*.whl + ./macos-arm64-libchdb.tar.gz + ./macos-arm64-libchdb-static.tar.gz + overwrite: true + - name: Upload pypi + if: startsWith(github.ref, 'refs/tags/v') + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv shell 3.8 + python -m twine upload dist/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/build_macos_x86_wheels.yml b/.github/workflows/build_macos_x86_wheels.yml index e42a743d180..8217b1ad033 100644 --- a/.github/workflows/build_macos_x86_wheels.yml +++ b/.github/workflows/build_macos_x86_wheels.yml @@ -1,298 +1,298 @@ -# name: Build macOS X86 +name: Build macOS X86 -# on: -# workflow_dispatch: -# inputs: -# TAG_NAME: -# description: 'Release Version Tag' -# required: true -# release: -# types: [created] -# push: -# branches: -# - main -# paths-ignore: -# - '**/*.md' -# pull_request: -# branches: -# - main -# paths-ignore: -# - '**/*.md' +on: + workflow_dispatch: + inputs: + TAG_NAME: + description: 'Release Version Tag' + required: true + release: + types: [created] + push: + branches: + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - main + paths-ignore: + - '**/*.md' -# jobs: -# build_universal_wheel: -# name: Build Universal Wheel (macOS x86_64) -# runs-on: macos-14-large -# steps: -# - name: Setup pyenv -# run: | -# curl https://pyenv.run | bash -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" +jobs: + build_universal_wheel: + name: Build Universal Wheel (macOS x86_64) + runs-on: macos-14-large + steps: + - name: Setup pyenv + run: | + curl https://pyenv.run | bash + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" -# pyenv install 3.8:latest -# pyenv install 3.9:latest -# pyenv install 3.10:latest -# pyenv install 3.11:latest -# pyenv install 3.12:latest -# pyenv install 3.13:latest -# pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 + pyenv install 3.8:latest + pyenv install 3.9:latest + pyenv install 3.10:latest + pyenv install 3.11:latest + pyenv install 3.12:latest + pyenv install 3.13:latest + pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 -# echo "Installed versions:" -# pyenv versions -# - name: Verify pyenv installations -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# echo "Installed Python versions:" -# pyenv versions -# echo "" -# echo "Verifying all required Python versions are available:" -# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do -# if ! pyenv versions --bare | grep -q "^$version"; then -# echo "ERROR: Python $version is not installed!" -# exit 1 -# fi -# echo "✓ Python $version is installed" -# done -# echo "All Python versions verified successfully!" -# - name: Install dependencies for all Python versions -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do -# echo "Installing dependencies for Python $version" -# pyenv shell $version -# python -m pip install --upgrade pip -# python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel>=0.40.0 -# pyenv shell --unset -# done -# - name: Remove /usr/local/bin/python3 -# run: | -# sudo rm -f /usr/local/bin/python3 -# - name: Install clang++ for macOS -# run: | -# pwd -# uname -a -# export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 -# brew update -# brew install ca-certificates lz4 mpdecimal openssl@3 readline sqlite xz z3 zstd -# brew install --ignore-dependencies llvm@19 -# brew install git ninja libtool gettext gcc binutils grep findutils nasm lld@19 libiconv -# brew install ccache || echo "ccache installation failed, continuing without it" -# brew install go -# cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm -# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:$PATH -# which clang++ -# clang++ --version -# which go -# go version -# ccache -s || echo "ccache not available yet" -# - name: Scan SQLite vulnerabilities with grype -# run: | -# # Install grype -# curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin -# # Update grype vulnerability database -# grype db update -# # Check SQLite vulnerabilities in Homebrew packages -# echo "Scanning SQLite packages for vulnerabilities..." -# GRYPE_RAW_OUTPUT=$(grype dir:/opt/homebrew --scope all-layers 2>/dev/null || true) -# echo "Raw grype output:" -# echo "$GRYPE_RAW_OUTPUT" -# SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) -# if [ -n "$SQLITE_SCAN_OUTPUT" ]; then -# echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." -# echo "SQLite vulnerability details:" -# echo "$SQLITE_SCAN_OUTPUT" -# exit 1 -# else -# echo "✅ No SQLite vulnerabilities found" -# fi -# continue-on-error: false -# - name: Upgrade Rust toolchain -# run: | -# rustup toolchain install nightly-2025-07-07 -# rustup default nightly-2025-07-07 -# rustup component add rust-src -# rustc --version -# cargo --version -# - uses: actions/checkout@v3 -# with: -# fetch-depth: 0 -# - name: Update submodules -# run: | -# git submodule update --init --recursive --jobs 4 -# - name: Run chdb/build.sh -# timeout-minutes: 600 -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# source ~/.cargo/env -# pyenv shell 3.8 -# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin -# export CC=$(brew --prefix llvm@19)/bin/clang -# export CXX=$(brew --prefix llvm@19)/bin/clang++ -# bash gen_manifest.sh -# bash ./chdb/build.sh -# pyenv shell 3.8 -# bash -x ./chdb/test_smoke.sh -# continue-on-error: false -# - name: Run chdb/build/build_static_lib.sh -# timeout-minutes: 600 -# run: | -# export PATH=$HOME/.pyenv/bin:$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin -# export CC=$(brew --prefix llvm@19)/bin/clang -# export CXX=$(brew --prefix llvm@19)/bin/clang++ -# source ~/.cargo/env -# eval "$(pyenv init -)" -# pyenv shell 3.8 -# bash ./chdb/build/build_static_lib.sh -# pyenv shell --unset -# continue-on-error: false -# - name: Debug libchdb -# run: | -# ls -lh -# llvm-nm libchdb.so | grep query_stable || true -# echo "Global Symbol in libchdb.so:" -# llvm-nm -g libchdb.so || true -# echo "Global Symbol in libclickhouse-local-chdb.a:" -# llvm-nm -g buildlib/programs/local/libclickhouse-local-chdb.a || true -# echo "Global Symbol in libclickhouse-local-lib.a:" -# llvm-nm -g buildlib/programs/local/libclickhouse-local-lib.a || true -# echo "pychdb_cmd.sh:" -# cat buildlib/pychdb_cmd.sh -# echo "libchdb_cmd.sh:" -# cat buildlib/libchdb_cmd.sh -# - name: Scan chdb libraries with grype -# run: | -# echo "Scanning chdb libraries for vulnerabilities..." -# # Files to scan -# FILES_TO_SCAN="" -# [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" -# [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" -# FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)" -# SQLITE_VULNERABILITIES_FOUND=false -# for file in $FILES_TO_SCAN; do -# if [ -f "$file" ]; then -# echo "=== Scanning $file ===" -# SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) -# echo "$SCAN_OUTPUT" -# if echo "$SCAN_OUTPUT" | grep -qi sqlite; then -# echo "❌ SQLite vulnerability found in $file" -# SQLITE_VULNERABILITIES_FOUND=true -# fi -# fi -# done -# if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then -# echo "❌ SQLite vulnerabilities detected in chdb libraries!" -# exit 1 -# else -# echo "✅ No SQLite vulnerabilities found in chdb libraries" -# fi -# continue-on-error: false -# - name: Run libchdb stub in examples dir -# run: | -# bash -x ./examples/runStub.sh -# - name: Build wheels -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# source ~/.cargo/env -# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin -# export CC=$(brew --prefix llvm@19)/bin/clang -# export CXX=$(brew --prefix llvm@19)/bin/clang++ -# pyenv shell 3.8 -# make wheel -# - name: Fix wheel platform tag -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# pyenv shell 3.8 -# python -m wheel tags --platform-tag=macosx_10_15_x86_64 --remove dist/*.whl -# - name: Setup core dump collection -# run: | -# mkdir -p tmp/core -# sudo sysctl kern.corefile=$PWD/tmp/core/core.%P -# sudo sysctl kern.coredump=1 -# ulimit -c unlimited -# - name: Test wheel on all Python versions -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# for version in 3.8 3.9 3.10 3.11 3.12 3.13; do -# echo "Testing chdb on Python $version" -# pyenv shell $version -# python -m pip install dist/*.whl --force-reinstall -# python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" -# make test -# pyenv shell --unset -# done -# continue-on-error: false -# - name: Check and upload core files if present -# if: always() -# run: | -# if ls tmp/core/core.* >/dev/null 2>&1; then -# echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV -# tar -czvf core-files-macos-x86_64.tar.gz tmp/core/core.* -# echo "Core files tar created: core-files-macos-x86_64.tar.gz" -# ls -lh core-files-macos-x86_64.tar.gz -# else -# echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV -# echo "No core files found in tmp/core" -# fi -# continue-on-error: true -# - name: Upload core files artifact -# if: always() && env.CORE_FILES_FOUND == 'true' -# uses: actions/upload-artifact@v4 -# with: -# name: core-files-macos-x86_64 -# path: core-files-macos-x86_64.tar.gz -# - name: Show files -# run: ls -lh dist -# shell: bash -# - name: Upload wheels to release -# if: startsWith(github.ref, 'refs/tags/v') -# run: | -# gh release upload ${{ github.ref_name }} dist/*.whl --clobber -# env: -# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} -# - name: Packege libchdb.so -# run: | -# cp programs/local/chdb.h chdb.h -# cp programs/local/chdb.hpp chdb.hpp -# tar -czvf macos-x86_64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp -# - name: Package libchdb.a -# run: | -# cp programs/local/chdb.h chdb.h -# cp programs/local/chdb.hpp chdb.hpp -# tar -czvf macos-x86_64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp -# - name: Upload libchdb.so to release -# if: startsWith(github.ref, 'refs/tags/v') -# run: | -# gh release upload ${{ github.ref_name }} macos-x86_64-libchdb.tar.gz --clobber -# env: -# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} -# - name: Upload libchdb.a to release -# if: startsWith(github.ref, 'refs/tags/v') -# run: | -# gh release upload ${{ github.ref_name }} macos-x86_64-libchdb-static.tar.gz --clobber -# env: -# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} -# - uses: actions/upload-artifact@v4 -# with: -# name: chdb-artifacts-macos-x86_64 -# path: | -# ./dist/*.whl -# ./macos-x86_64-libchdb.tar.gz -# ./macos-x86_64-libchdb-static.tar.gz -# overwrite: true -# - name: Upload pypi -# if: startsWith(github.ref, 'refs/tags/v') -# run: | -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" -# pyenv shell 3.8 -# python -m twine upload dist/*.whl -# env: -# TWINE_USERNAME: __token__ -# TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + echo "Installed versions:" + pyenv versions + - name: Verify pyenv installations + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + echo "Installed Python versions:" + pyenv versions + echo "" + echo "Verifying all required Python versions are available:" + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + if ! pyenv versions --bare | grep -q "^$version"; then + echo "ERROR: Python $version is not installed!" + exit 1 + fi + echo "✓ Python $version is installed" + done + echo "All Python versions verified successfully!" + - name: Install dependencies for all Python versions + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + echo "Installing dependencies for Python $version" + pyenv shell $version + python -m pip install --upgrade pip + python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel>=0.40.0 + pyenv shell --unset + done + - name: Remove /usr/local/bin/python3 + run: | + sudo rm -f /usr/local/bin/python3 + - name: Install clang++ for macOS + run: | + pwd + uname -a + export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 + brew update + brew install ca-certificates lz4 mpdecimal openssl@3 readline sqlite xz z3 zstd + brew install --ignore-dependencies llvm@19 + brew install git ninja libtool gettext gcc binutils grep findutils nasm lld@19 libiconv + brew install ccache || echo "ccache installation failed, continuing without it" + brew install go + cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm + export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:$PATH + which clang++ + clang++ --version + which go + go version + ccache -s || echo "ccache not available yet" + - name: Scan SQLite vulnerabilities with grype + run: | + # Install grype + curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin + # Update grype vulnerability database + grype db update + # Check SQLite vulnerabilities in Homebrew packages + echo "Scanning SQLite packages for vulnerabilities..." + GRYPE_RAW_OUTPUT=$(grype dir:/opt/homebrew --scope all-layers 2>/dev/null || true) + echo "Raw grype output:" + echo "$GRYPE_RAW_OUTPUT" + SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) + if [ -n "$SQLITE_SCAN_OUTPUT" ]; then + echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." + echo "SQLite vulnerability details:" + echo "$SQLITE_SCAN_OUTPUT" + exit 1 + else + echo "✅ No SQLite vulnerabilities found" + fi + continue-on-error: false + - name: Upgrade Rust toolchain + run: | + rustup toolchain install nightly-2025-07-07 + rustup default nightly-2025-07-07 + rustup component add rust-src + rustc --version + cargo --version + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Update submodules + run: | + git submodule update --init --recursive --jobs 4 + - name: Run chdb/build.sh + timeout-minutes: 600 + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + source ~/.cargo/env + pyenv shell 3.8 + export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin + export CC=$(brew --prefix llvm@19)/bin/clang + export CXX=$(brew --prefix llvm@19)/bin/clang++ + bash gen_manifest.sh + bash ./chdb/build.sh + pyenv shell 3.8 + bash -x ./chdb/test_smoke.sh + continue-on-error: false + - name: Run chdb/build/build_static_lib.sh + timeout-minutes: 600 + run: | + export PATH=$HOME/.pyenv/bin:$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin + export CC=$(brew --prefix llvm@19)/bin/clang + export CXX=$(brew --prefix llvm@19)/bin/clang++ + source ~/.cargo/env + eval "$(pyenv init -)" + pyenv shell 3.8 + bash ./chdb/build/build_static_lib.sh + pyenv shell --unset + continue-on-error: false + - name: Debug libchdb + run: | + ls -lh + llvm-nm libchdb.so | grep query_stable || true + echo "Global Symbol in libchdb.so:" + llvm-nm -g libchdb.so || true + echo "Global Symbol in libclickhouse-local-chdb.a:" + llvm-nm -g buildlib/programs/local/libclickhouse-local-chdb.a || true + echo "Global Symbol in libclickhouse-local-lib.a:" + llvm-nm -g buildlib/programs/local/libclickhouse-local-lib.a || true + echo "pychdb_cmd.sh:" + cat buildlib/pychdb_cmd.sh + echo "libchdb_cmd.sh:" + cat buildlib/libchdb_cmd.sh + - name: Scan chdb libraries with grype + run: | + echo "Scanning chdb libraries for vulnerabilities..." + # Files to scan + FILES_TO_SCAN="" + [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" + [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" + FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)" + SQLITE_VULNERABILITIES_FOUND=false + for file in $FILES_TO_SCAN; do + if [ -f "$file" ]; then + echo "=== Scanning $file ===" + SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) + echo "$SCAN_OUTPUT" + if echo "$SCAN_OUTPUT" | grep -qi sqlite; then + echo "❌ SQLite vulnerability found in $file" + SQLITE_VULNERABILITIES_FOUND=true + fi + fi + done + if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then + echo "❌ SQLite vulnerabilities detected in chdb libraries!" + exit 1 + else + echo "✅ No SQLite vulnerabilities found in chdb libraries" + fi + continue-on-error: false + - name: Run libchdb stub in examples dir + run: | + bash -x ./examples/runStub.sh + - name: Build wheels + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + source ~/.cargo/env + export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin + export CC=$(brew --prefix llvm@19)/bin/clang + export CXX=$(brew --prefix llvm@19)/bin/clang++ + pyenv shell 3.8 + make wheel + - name: Fix wheel platform tag + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv shell 3.8 + python -m wheel tags --platform-tag=macosx_10_15_x86_64 --remove dist/*.whl + - name: Setup core dump collection + run: | + mkdir -p tmp/core + sudo sysctl kern.corefile=$PWD/tmp/core/core.%P + sudo sysctl kern.coredump=1 + ulimit -c unlimited + - name: Test wheel on all Python versions + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do + echo "Testing chdb on Python $version" + pyenv shell $version + python -m pip install dist/*.whl --force-reinstall + python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" + make test + pyenv shell --unset + done + continue-on-error: false + - name: Check and upload core files if present + if: always() + run: | + if ls tmp/core/core.* >/dev/null 2>&1; then + echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV + tar -czvf core-files-macos-x86_64.tar.gz tmp/core/core.* + echo "Core files tar created: core-files-macos-x86_64.tar.gz" + ls -lh core-files-macos-x86_64.tar.gz + else + echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV + echo "No core files found in tmp/core" + fi + continue-on-error: true + - name: Upload core files artifact + if: always() && env.CORE_FILES_FOUND == 'true' + uses: actions/upload-artifact@v4 + with: + name: core-files-macos-x86_64 + path: core-files-macos-x86_64.tar.gz + - name: Show files + run: ls -lh dist + shell: bash + - name: Upload wheels to release + if: startsWith(github.ref, 'refs/tags/v') + run: | + gh release upload ${{ github.ref_name }} dist/*.whl --clobber + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + - name: Packege libchdb.so + run: | + cp programs/local/chdb.h chdb.h + cp programs/local/chdb.hpp chdb.hpp + tar -czvf macos-x86_64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp + - name: Package libchdb.a + run: | + cp programs/local/chdb.h chdb.h + cp programs/local/chdb.hpp chdb.hpp + tar -czvf macos-x86_64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp + - name: Upload libchdb.so to release + if: startsWith(github.ref, 'refs/tags/v') + run: | + gh release upload ${{ github.ref_name }} macos-x86_64-libchdb.tar.gz --clobber + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + - name: Upload libchdb.a to release + if: startsWith(github.ref, 'refs/tags/v') + run: | + gh release upload ${{ github.ref_name }} macos-x86_64-libchdb-static.tar.gz --clobber + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + - uses: actions/upload-artifact@v4 + with: + name: chdb-artifacts-macos-x86_64 + path: | + ./dist/*.whl + ./macos-x86_64-libchdb.tar.gz + ./macos-x86_64-libchdb-static.tar.gz + overwrite: true + - name: Upload pypi + if: startsWith(github.ref, 'refs/tags/v') + run: | + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv shell 3.8 + python -m twine upload dist/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index a6027b99efa..c4561d3ada3 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -103,19 +103,16 @@ jobs: eval "$(pyenv init -)" # Install Python versions - # for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - for version in 3.8 3.9; do + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do echo "Installing Python $version" pyenv install $version:latest done - # pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 - pyenv global 3.8 3.9 + pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 # Verify installations echo "Installed versions:" pyenv versions - # for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - for version in 3.8 3.9; do + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do if ! pyenv versions --bare | grep -q "^$version"; then echo "ERROR: Python $version is not installed!" exit 1 @@ -131,16 +128,14 @@ jobs: rustup component add --toolchain nightly-2025-07-07 rust-src # Install Python dependencies - # for version in 3.8 3.9 3.10 3.11 3.12 3.13; do - for version in 3.8 3.9; do + for version in 3.8 3.9 3.10 3.11 3.12 3.13; do echo "Installing dependencies for Python $version" pyenv shell $version python -m pip install --upgrade pip if [ "$version" = "3.8" ]; then python -m pip install setuptools tox twine psutil wheel else - # python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel - python -m pip install setuptools tox pandas pyarrow twine psutil wheel + python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel fi pyenv shell --unset done diff --git a/contrib/pybind11-cmake/CMakeLists.txt b/contrib/pybind11-cmake/CMakeLists.txt index 398c6aa71a8..0427cf47b71 100644 --- a/contrib/pybind11-cmake/CMakeLists.txt +++ b/contrib/pybind11-cmake/CMakeLists.txt @@ -78,7 +78,6 @@ if(Python_FOUND) target_link_libraries(${PYBIND11_NONLIMITEDAPI_LIBNAME} PUBLIC ch_contrib::pybind11_stubs) if (USE_MUSL AND CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") - # Force linking entire libgcc archive to ensure __floatunsitf is included target_link_libraries(${PYBIND11_NONLIMITEDAPI_LIBNAME} PRIVATE -Wl,--whole-archive -lgcc -Wl,--no-whole-archive) message(STATUS "Linking whole libgcc archive for ARM64+musl to provide all compiler runtime symbols") endif() diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index c170232bc94..f3d869e10d2 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -71,17 +71,6 @@ # include #endif -#if defined(USE_MUSL) && defined(__aarch64__) -void local_server_musl_compile_stub(int arg) -{ - jmp_buf buf1; - sigjmp_buf buf2; - - setjmp(buf1); - sigsetjmp(buf2, arg); -} -#endif - namespace fs = std::filesystem; namespace CurrentMetrics diff --git a/src/TableFunctions/ITableFunction.cpp b/src/TableFunctions/ITableFunction.cpp index d71e1be8aaf..a5b53d3a8a7 100644 --- a/src/TableFunctions/ITableFunction.cpp +++ b/src/TableFunctions/ITableFunction.cpp @@ -7,17 +7,6 @@ #include #include -#if defined(USE_MUSL) && defined(__aarch64__) -void i_table_function_musl_compile_stub(int arg) -{ - jmp_buf buf1; - sigjmp_buf buf2; - - setjmp(buf1); - sigsetjmp(buf2, arg); -} -#endif - namespace ProfileEvents { extern const Event TableFunctionExecute; From 9ff397b59d058fe6ea37acd65c8f004dbe2b5bd4 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Wed, 15 Oct 2025 10:45:39 +0800 Subject: [PATCH 36/41] chore: update build.sh --- chdb/build.sh | 139 ++++++++++++++++++++++++-------------------------- 1 file changed, 68 insertions(+), 71 deletions(-) diff --git a/chdb/build.sh b/chdb/build.sh index 7cf58634308..f31e904e964 100755 --- a/chdb/build.sh +++ b/chdb/build.sh @@ -3,7 +3,7 @@ set -e # default to build Release -build_type=${1:-Debug} +build_type=${1:-Release} DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" @@ -52,9 +52,9 @@ if [ "$(uname)" == "Darwin" ]; then fi fi elif [ "$(uname)" == "Linux" ]; then - # GLIBC_COMPATIBILITY="-DGLIBC_COMPATIBILITY=1" + GLIBC_COMPATIBILITY="-DGLIBC_COMPATIBILITY=1" UNWIND="-DUSE_UNWIND=1" - JEMALLOC="-DENABLE_JEMALLOC=0" + JEMALLOC="-DENABLE_JEMALLOC=1" PYINIT_ENTRY="-Wl,-ePyInit_${CHDB_PY_MOD}" ICU="-DENABLE_ICU=1" SED_INPLACE="sed -i" @@ -96,9 +96,6 @@ CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${build_type} -DENABLE_THINLTO=0 -DENABLE_TESTS=0 -DENABLE_KAFKA=1 -DENABLE_LIBPQXX=1 -DENABLE_NATS=0 -DENABLE_AMQPCPP=0 -DENABLE_NURAFT=0 \ -DENABLE_CASSANDRA=0 -DENABLE_ODBC=0 -DENABLE_NLP=0 \ -DENABLE_LDAP=0 \ - -DUSE_MUSL=1 \ - -DRust_CARGO_TARGET=x86_64-unknown-linux-musl \ - -DRust_RUSTUP_INSTALL_MISSING_TARGET=ON \ ${MYSQL} \ ${HDFS} \ -DENABLE_LIBRARIES=0 ${RUST_FEATURES} \ @@ -174,71 +171,71 @@ CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${build_type} -DENABLE_THINLTO=0 -DENABLE_TESTS=0 # fi -# LIBCHDB_SO="libchdb.so" +LIBCHDB_SO="libchdb.so" # Build libchdb.so -# cmake ${CMAKE_ARGS} -DENABLE_PYTHON=0 .. -# ninja -d keeprsp +cmake ${CMAKE_ARGS} -DENABLE_PYTHON=0 .. +ninja -d keeprsp BINARY=${BUILD_DIR}/programs/clickhouse -# echo -e "\nBINARY: ${BINARY}" -# ls -lh ${BINARY} -# echo -e "\nldd ${BINARY}" -# ${LDD} ${BINARY} || echo "Binary is statically linked (not a dynamic executable)" -# rm -f ${BINARY} - -# cd ${BUILD_DIR} -# ninja -d keeprsp -v > build.log || true -# USING_RESPONSE_FILE=$(grep -m 1 'clang++.*-o programs/clickhouse .*' build.log | grep '@CMakeFiles/clickhouse.rsp' || true) - -# if [ ! "${USING_RESPONSE_FILE}" == "" ]; then -# if [ -f CMakeFiles/clickhouse.rsp ]; then -# cp -a CMakeFiles/clickhouse.rsp CMakeFiles/libchdb.rsp -# else -# echo "CMakeFiles/clickhouse.rsp not found" -# exit 1 -# fi -# fi +echo -e "\nBINARY: ${BINARY}" +ls -lh ${BINARY} +echo -e "\nldd ${BINARY}" +${LDD} ${BINARY} +rm -f ${BINARY} -# LIBCHDB_CMD=$(grep -m 1 'clang++.*-o programs/clickhouse .*' build.log \ -# | sed "s/-o programs\/clickhouse/-fPIC -shared -o ${LIBCHDB_SO}/" \ -# | sed 's/^[^&]*&& //' | sed 's/&&.*//' \ -# | sed 's/ -Wl,-undefined,error/ -Wl,-undefined,dynamic_lookup/g' \ -# | sed 's/ -Xlinker --no-undefined//g' \ -# | sed 's/@CMakeFiles\/clickhouse.rsp/@CMakeFiles\/libchdb.rsp/g' \ -# ) +cd ${BUILD_DIR} +ninja -d keeprsp -v > build.log || true +USING_RESPONSE_FILE=$(grep -m 1 'clang++.*-o programs/clickhouse .*' build.log | grep '@CMakeFiles/clickhouse.rsp' || true) -# # generate the command to generate libchdb.so -# LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed 's/ '${CHDB_PY_MODULE}'/ '${LIBCHDB_SO}'/g') +if [ ! "${USING_RESPONSE_FILE}" == "" ]; then + if [ -f CMakeFiles/clickhouse.rsp ]; then + cp -a CMakeFiles/clickhouse.rsp CMakeFiles/libchdb.rsp + else + echo "CMakeFiles/clickhouse.rsp not found" + exit 1 + fi +fi -# if [ ! "${USING_RESPONSE_FILE}" == "" ]; then -# ${SED_INPLACE} 's/ '${CHDB_PY_MODULE}'/ '${LIBCHDB_SO}'/g' CMakeFiles/libchdb.rsp -# fi +LIBCHDB_CMD=$(grep -m 1 'clang++.*-o programs/clickhouse .*' build.log \ + | sed "s/-o programs\/clickhouse/-fPIC -shared -o ${LIBCHDB_SO}/" \ + | sed 's/^[^&]*&& //' | sed 's/&&.*//' \ + | sed 's/ -Wl,-undefined,error/ -Wl,-undefined,dynamic_lookup/g' \ + | sed 's/ -Xlinker --no-undefined//g' \ + | sed 's/@CMakeFiles\/clickhouse.rsp/@CMakeFiles\/libchdb.rsp/g' \ + ) -# if [ "$(uname)" == "Linux" ]; then -# LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed 's/ '${PYINIT_ENTRY}'/ /g') -# if [ ! "${USING_RESPONSE_FILE}" == "" ]; then -# ${SED_INPLACE} 's/ '${PYINIT_ENTRY}'/ /g' CMakeFiles/libchdb.rsp -# fi -# fi +# generate the command to generate libchdb.so +LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed 's/ '${CHDB_PY_MODULE}'/ '${LIBCHDB_SO}'/g') -# if [ "$(uname)" == "Darwin" ]; then -# LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed 's/ '${PYINIT_ENTRY}'/ -Wl,-exported_symbol,_query_stable -Wl,-exported_symbol,_free_result -Wl,-exported_symbol,_query_stable_v2 -Wl,-exported_symbol,_free_result_v2/g') -# # ${SED_INPLACE} 's/ '${PYINIT_ENTRY}'/ -Wl,-exported_symbol,_query_stable -Wl,-exported_symbol,_free_result -Wl,-exported_symbol,_query_stable_v2 -Wl,-exported_symbol,_free_result_v2/g' CMakeFiles/libchdb.rsp -# fi +if [ ! "${USING_RESPONSE_FILE}" == "" ]; then + ${SED_INPLACE} 's/ '${CHDB_PY_MODULE}'/ '${LIBCHDB_SO}'/g' CMakeFiles/libchdb.rsp +fi + +if [ "$(uname)" == "Linux" ]; then + LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed 's/ '${PYINIT_ENTRY}'/ /g') + if [ ! "${USING_RESPONSE_FILE}" == "" ]; then + ${SED_INPLACE} 's/ '${PYINIT_ENTRY}'/ /g' CMakeFiles/libchdb.rsp + fi +fi + +if [ "$(uname)" == "Darwin" ]; then + LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed 's/ '${PYINIT_ENTRY}'/ -Wl,-exported_symbol,_query_stable -Wl,-exported_symbol,_free_result -Wl,-exported_symbol,_query_stable_v2 -Wl,-exported_symbol,_free_result_v2/g') + # ${SED_INPLACE} 's/ '${PYINIT_ENTRY}'/ -Wl,-exported_symbol,_query_stable -Wl,-exported_symbol,_free_result -Wl,-exported_symbol,_query_stable_v2 -Wl,-exported_symbol,_free_result_v2/g' CMakeFiles/libchdb.rsp +fi -# LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed 's/@CMakeFiles\/clickhouse.rsp/@CMakeFiles\/libchdb.rsp/g') +LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed 's/@CMakeFiles\/clickhouse.rsp/@CMakeFiles\/libchdb.rsp/g') -# # Step 4: -# # save the command to a file for debug -# echo ${LIBCHDB_CMD} > libchdb_cmd.sh +# Step 4: +# save the command to a file for debug +echo ${LIBCHDB_CMD} > libchdb_cmd.sh -# # Step 5: -# ${LIBCHDB_CMD} +# Step 5: +${LIBCHDB_CMD} LIBCHDB_DIR=${BUILD_DIR}/ -# LIBCHDB=${LIBCHDB_DIR}/${LIBCHDB_SO} -# ls -lh ${LIBCHDB} +LIBCHDB=${LIBCHDB_DIR}/${LIBCHDB_SO} +ls -lh ${LIBCHDB} # build chdb python module py_version="3.8" @@ -312,43 +309,43 @@ ls -lh ${CHDB_PY_MODULE} LIBCHDB_DIR=${BUILD_DIR}/ PYCHDB=${LIBCHDB_DIR}/${CHDB_PY_MODULE} -# LIBCHDB=${LIBCHDB_DIR}/${LIBCHDB_SO} +LIBCHDB=${LIBCHDB_DIR}/${LIBCHDB_SO} if [ ${build_type} == "Debug" ]; then echo -e "\nDebug build, skip strip" else echo -e "\nStrip the binary:" ${STRIP} --remove-section=.comment --remove-section=.note ${PYCHDB} - # ${STRIP} --remove-section=.comment --remove-section=.note ${LIBCHDB} + ${STRIP} --remove-section=.comment --remove-section=.note ${LIBCHDB} fi echo -e "\nStripe the binary:" echo -e "\nPYCHDB: ${PYCHDB}" ls -lh ${PYCHDB} -# echo -e "\nLIBCHDB: ${LIBCHDB}" -# ls -lh ${LIBCHDB} +echo -e "\nLIBCHDB: ${LIBCHDB}" +ls -lh ${LIBCHDB} echo -e "\nldd ${PYCHDB}" -${LDD} ${PYCHDB} || echo "Binary is statically linked (not a dynamic executable)" +${LDD} ${PYCHDB} echo -e "\nfile info of ${PYCHDB}" file ${PYCHDB} -# echo -e "\nldd ${LIBCHDB}" -# ${LDD} ${LIBCHDB} || echo "Binary is statically linked (not a dynamic executable)" -# echo -e "\nfile info of ${LIBCHDB}" -# file ${LIBCHDB} +echo -e "\nldd ${LIBCHDB}" +${LDD} ${LIBCHDB} +echo -e "\nfile info of ${LIBCHDB}" +file ${LIBCHDB} rm -f ${CHDB_DIR}/*.so cp -a ${PYCHDB} ${CHDB_DIR}/${CHDB_PY_MODULE} -# cp -a ${LIBCHDB} ${PROJ_DIR}/${LIBCHDB_SO} +cp -a ${LIBCHDB} ${PROJ_DIR}/${LIBCHDB_SO} echo -e "\nSymbols:" echo -e "\nPyInit in PYCHDB: ${PYCHDB}" ${NM} ${PYCHDB} | grep PyInit || true -# echo -e "\nPyInit in LIBCHDB: ${LIBCHDB}" -# ${NM} ${LIBCHDB} | grep PyInit || echo "PyInit not found in ${LIBCHDB}, it's OK" +echo -e "\nPyInit in LIBCHDB: ${LIBCHDB}" +${NM} ${LIBCHDB} | grep PyInit || echo "PyInit not found in ${LIBCHDB}, it's OK" echo -e "\nquery_stable in PYCHDB: ${PYCHDB}" ${NM} ${PYCHDB} | grep query_stable || true -# echo -e "\nquery_stable in LIBCHDB: ${LIBCHDB}" -# ${NM} ${LIBCHDB} | grep query_stable || true +echo -e "\nquery_stable in LIBCHDB: ${LIBCHDB}" +${NM} ${LIBCHDB} | grep query_stable || true echo -e "\nAfter copy:" cd ${PROJ_DIR} && pwd From 7ab5917f02e644cbc7a99ca1b4b43f5cdb45b910 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Wed, 15 Oct 2025 18:28:57 +0800 Subject: [PATCH 37/41] chore: add cpu check --- .github/workflows/build_musllinux_x86_wheels.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/build_musllinux_x86_wheels.yml b/.github/workflows/build_musllinux_x86_wheels.yml index c48a803f97c..53c7f63b418 100644 --- a/.github/workflows/build_musllinux_x86_wheels.yml +++ b/.github/workflows/build_musllinux_x86_wheels.yml @@ -38,6 +38,16 @@ jobs: else echo "Not musl libc" fi + + echo "=== CPU Information ===" + cat /proc/cpuinfo + echo "" + echo "=== Checking CPU requirements ===" + if grep -P "^(?=.*ssse3)(?=.*sse4_1)(?=.*sse4_2)" /proc/cpuinfo > /dev/null; then + echo "CPU meets minimum requirements" + else + echo "CPU does not meet minimum requirements" + fi - name: Install Python build dependencies run: | apk update From 98b02a261ec0ef2c04b9c8b6460f189814fa02d7 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Wed, 15 Oct 2025 20:40:15 +0800 Subject: [PATCH 38/41] chore: update workflow --- .github/workflows/build_musllinux_x86_wheels.yml | 2 +- cmake/cpu_features.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_musllinux_x86_wheels.yml b/.github/workflows/build_musllinux_x86_wheels.yml index 53c7f63b418..3797060cefd 100644 --- a/.github/workflows/build_musllinux_x86_wheels.yml +++ b/.github/workflows/build_musllinux_x86_wheels.yml @@ -43,7 +43,7 @@ jobs: cat /proc/cpuinfo echo "" echo "=== Checking CPU requirements ===" - if grep -P "^(?=.*ssse3)(?=.*sse4_1)(?=.*sse4_2)" /proc/cpuinfo > /dev/null; then + if grep -q "ssse3" /proc/cpuinfo && grep -q "sse4_1" /proc/cpuinfo && grep -q "sse4_2" /proc/cpuinfo; then echo "CPU meets minimum requirements" else echo "CPU does not meet minimum requirements" diff --git a/cmake/cpu_features.cmake b/cmake/cpu_features.cmake index 264157aa86c..a2fa25c7356 100644 --- a/cmake/cpu_features.cmake +++ b/cmake/cpu_features.cmake @@ -156,7 +156,7 @@ elseif (ARCH_AMD64) endif() # Same best-effort check for x86 as above for ARM. - if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "amd64|x86_64" AND NOT NO_SSE3_OR_HIGHER AND NOT USE_MUSL) + if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "amd64|x86_64" AND NOT NO_SSE3_OR_HIGHER) # Test for flags in standard profile but not in NO_SSE3_OR_HIGHER profile. # /proc/cpuid for Intel Xeon 8124: "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse # sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf From 3f3c26cc1773964dda365a94c292916460329422 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Thu, 16 Oct 2025 00:33:30 +0800 Subject: [PATCH 39/41] chore: update cpu_features.cmake --- cmake/cpu_features.cmake | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/cmake/cpu_features.cmake b/cmake/cpu_features.cmake index a2fa25c7356..a70d523adcb 100644 --- a/cmake/cpu_features.cmake +++ b/cmake/cpu_features.cmake @@ -98,14 +98,20 @@ elseif (ARCH_AARCH64) # and the build machine is too old, i.e. doesn't satisfy above modern profile, then these intermediate binaries will not run (dump # SIGILL). Even if they could run, the build machine wouldn't be able to run the ClickHouse binary. In that case, suggest to run the # build with the compat profile. - if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)" AND NOT NO_ARMV81_OR_HIGHER AND NOT USE_MUSL) + if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)" AND NOT NO_ARMV81_OR_HIGHER) # CPU features in /proc/cpuinfo and compiler flags don't align :( ... pick some obvious flags contained in the modern but not in the # legacy profile (full Graviton 3 /proc/cpuinfo is "fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm # jscvt fcma lrcpc dcpop sha3 sm3 sm4 asimddp sha512 sve asimdfhm dit uscat ilrcpc flagm ssbs paca pacg dcpodp svei8mm svebf16 i8mm # bf16 dgh rng") - execute_process( - COMMAND grep -P "^(?=.*atomic)(?=.*ssbs)" /proc/cpuinfo - OUTPUT_VARIABLE FLAGS) + if (USE_MUSL) + execute_process( + COMMAND grep -q "atomic" /proc/cpuinfo && grep -q "ssbs" /proc/cpuinfo + OUTPUT_VARIABLE FLAGS) + else() + execute_process( + COMMAND grep -P "^(?=.*atomic)(?=.*ssbs)" /proc/cpuinfo + OUTPUT_VARIABLE FLAGS) + endif() if (NOT FLAGS) MESSAGE(FATAL_ERROR "The build machine does not satisfy the minimum CPU requirements, try to run cmake with -DNO_ARMV81_OR_HIGHER=1") endif() @@ -163,9 +169,15 @@ elseif (ARCH_AMD64) # tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c # rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx # avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke"" - execute_process( - COMMAND grep -P "^(?=.*ssse3)(?=.*sse4_1)(?=.*sse4_2)" /proc/cpuinfo - OUTPUT_VARIABLE FLAGS) + if (USE_MUSL) + execute_process( + COMMAND grep -q "ssse3" /proc/cpuinfo && grep -q "sse4_1" /proc/cpuinfo && grep -q "sse4_2" /proc/cpuinfo + OUTPUT_VARIABLE FLAGS) + else() + execute_process( + COMMAND grep -P "^(?=.*ssse3)(?=.*sse4_1)(?=.*sse4_2)" /proc/cpuinfo + OUTPUT_VARIABLE FLAGS) + endif() if (NOT FLAGS) MESSAGE(FATAL_ERROR "The build machine does not satisfy the minimum CPU requirements, try to run cmake with -DNO_SSE3_OR_HIGHER=1") endif() From b5d7020cb5aca36164c6e3e83a6565586343d983 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Thu, 16 Oct 2025 01:28:03 +0800 Subject: [PATCH 40/41] Revert "chore: update cpu_features.cmake" This reverts commit 3f3c26cc1773964dda365a94c292916460329422. --- cmake/cpu_features.cmake | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/cmake/cpu_features.cmake b/cmake/cpu_features.cmake index a70d523adcb..a2fa25c7356 100644 --- a/cmake/cpu_features.cmake +++ b/cmake/cpu_features.cmake @@ -98,20 +98,14 @@ elseif (ARCH_AARCH64) # and the build machine is too old, i.e. doesn't satisfy above modern profile, then these intermediate binaries will not run (dump # SIGILL). Even if they could run, the build machine wouldn't be able to run the ClickHouse binary. In that case, suggest to run the # build with the compat profile. - if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)" AND NOT NO_ARMV81_OR_HIGHER) + if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)" AND NOT NO_ARMV81_OR_HIGHER AND NOT USE_MUSL) # CPU features in /proc/cpuinfo and compiler flags don't align :( ... pick some obvious flags contained in the modern but not in the # legacy profile (full Graviton 3 /proc/cpuinfo is "fp asimd evtstrm aes pmull sha1 sha2 crc32 atomics fphp asimdhp cpuid asimdrdm # jscvt fcma lrcpc dcpop sha3 sm3 sm4 asimddp sha512 sve asimdfhm dit uscat ilrcpc flagm ssbs paca pacg dcpodp svei8mm svebf16 i8mm # bf16 dgh rng") - if (USE_MUSL) - execute_process( - COMMAND grep -q "atomic" /proc/cpuinfo && grep -q "ssbs" /proc/cpuinfo - OUTPUT_VARIABLE FLAGS) - else() - execute_process( - COMMAND grep -P "^(?=.*atomic)(?=.*ssbs)" /proc/cpuinfo - OUTPUT_VARIABLE FLAGS) - endif() + execute_process( + COMMAND grep -P "^(?=.*atomic)(?=.*ssbs)" /proc/cpuinfo + OUTPUT_VARIABLE FLAGS) if (NOT FLAGS) MESSAGE(FATAL_ERROR "The build machine does not satisfy the minimum CPU requirements, try to run cmake with -DNO_ARMV81_OR_HIGHER=1") endif() @@ -169,15 +163,9 @@ elseif (ARCH_AMD64) # tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c # rdrand hypervisor lahf_lm abm 3dnowprefetch invpcid_single pti fsgsbase tsc_adjust bmi1 hle avx2 smep bmi2 erms invpcid rtm mpx # avx512f avx512dq rdseed adx smap clflushopt clwb avx512cd avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves ida arat pku ospke"" - if (USE_MUSL) - execute_process( - COMMAND grep -q "ssse3" /proc/cpuinfo && grep -q "sse4_1" /proc/cpuinfo && grep -q "sse4_2" /proc/cpuinfo - OUTPUT_VARIABLE FLAGS) - else() - execute_process( - COMMAND grep -P "^(?=.*ssse3)(?=.*sse4_1)(?=.*sse4_2)" /proc/cpuinfo - OUTPUT_VARIABLE FLAGS) - endif() + execute_process( + COMMAND grep -P "^(?=.*ssse3)(?=.*sse4_1)(?=.*sse4_2)" /proc/cpuinfo + OUTPUT_VARIABLE FLAGS) if (NOT FLAGS) MESSAGE(FATAL_ERROR "The build machine does not satisfy the minimum CPU requirements, try to run cmake with -DNO_SSE3_OR_HIGHER=1") endif() From 26a4da382a5025f79254e96d4e7781353d1ff03f Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Thu, 16 Oct 2025 01:29:31 +0800 Subject: [PATCH 41/41] chore: update cpu_features.cmake --- cmake/cpu_features.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cpu_features.cmake b/cmake/cpu_features.cmake index a2fa25c7356..264157aa86c 100644 --- a/cmake/cpu_features.cmake +++ b/cmake/cpu_features.cmake @@ -156,7 +156,7 @@ elseif (ARCH_AMD64) endif() # Same best-effort check for x86 as above for ARM. - if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "amd64|x86_64" AND NOT NO_SSE3_OR_HIGHER) + if (OS_LINUX AND CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "amd64|x86_64" AND NOT NO_SSE3_OR_HIGHER AND NOT USE_MUSL) # Test for flags in standard profile but not in NO_SSE3_OR_HIGHER profile. # /proc/cpuid for Intel Xeon 8124: "fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse # sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology nonstop_tsc cpuid aperfmperf