Merge pull request #22 from chdb-io/enable-jemalloc

Enable jemalloc in linux x86_64 build

.github/workflows/build_wheels.yml

@@ -74,7 +74,7 @@ jobs:
         with:
           key: ${{ matrix.os }}
           max-size: 5G
-          append-timestamp: false
+          append-timestamp: true
       - name: remove old clang and link clang-15 to clang
         if: matrix.os == 'ubuntu-20.04'
         run: |
@@ -199,7 +199,7 @@ jobs:
         with:
           key: ${{ matrix.os }}
           max-size: 5G
-          append-timestamp: false
+          append-timestamp: true
       - name: Run chdb/build.sh
         timeout-minutes: 300
         run: |
@@ -314,7 +314,7 @@ jobs:
         with:
           key: ${{ matrix.os }}
           max-size: 5G
-          append-timestamp: false
+          append-timestamp: true
       - name: Prepare chdb/build.sh
         run: |
           python3 -m pip install pybind11

.gitmodules

@@ -33,9 +33,6 @@
 [submodule "contrib/mariadb-connector-c"]
 	path = contrib/mariadb-connector-c
 	url = https://github.com/ClickHouse/mariadb-connector-c.git
-[submodule "contrib/jemalloc"]
-	path = contrib/jemalloc
-	url = https://github.com/jemalloc/jemalloc.git
 [submodule "contrib/unixodbc"]
 	path = contrib/unixodbc
 	url = https://github.com/ClickHouse/UnixODBC.git
@@ -294,3 +291,6 @@
 [submodule "contrib/google-benchmark"]
 	path = contrib/google-benchmark
 	url = https://github.com/google/benchmark.git
+[submodule "contrib/jemalloc"]
+	path = contrib/jemalloc
+	url = https://github.com/auxten/jemalloc.git

CMakeLists.txt

@@ -521,6 +521,9 @@ add_subdirectory (contrib EXCLUDE_FROM_ALL)
 
 if (NOT ENABLE_JEMALLOC)
     message (WARNING "Non default allocator is disabled. This is not recommended for production builds.")
+else ()
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_JEMALLOC=1 -DJEMALLOC_NO_RENAME=1")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DUSE_JEMALLOC=1 -DJEMALLOC_NO_RENAME=1")
 endif ()
 
 macro (clickhouse_add_executable target)

chdb/build.sh

@@ -12,6 +12,7 @@ BUILD_DIR=${PROJ_DIR}/buildlib
 if [ "$(uname)" == "Darwin" ]; then
     GLIBC_COMPATIBILITY="-DGLIBC_COMPATIBILITY=0"
     UNWIND="-DUSE_UNWIND=0"
+    JEMALLOC="-DENABLE_JEMALLOC=0"
     PYINIT_ENTRY="-Wl,-exported_symbol,_PyInit_${CHDB_PY_MOD}"
     # if Darwin ARM64 (M1, M2), disable AVX
     if [ "$(uname -m)" == "arm64" ]; then
@@ -42,6 +43,7 @@ if [ "$(uname)" == "Darwin" ]; then
 elif [ "$(uname)" == "Linux" ]; then
     GLIBC_COMPATIBILITY="-DGLIBC_COMPATIBILITY=1"
     UNWIND="-DUSE_UNWIND=1"
+    JEMALLOC="-DENABLE_JEMALLOC=1"
     PYINIT_ENTRY="-Wl,-ePyInit_${CHDB_PY_MOD}"
     AVX_SUPPORT="-DENABLE_AVX=1 -DENABLE_AVX2=0"
     EMBEDDED_COMPILER="-DENABLE_EMBEDDED_COMPILER=1"
@@ -67,7 +69,7 @@ cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_THINLTO=1 -DENABLE_TESTS=0 -DENABLE_CL
     ${GLIBC_COMPATIBILITY} \
     -DCLICKHOUSE_ONE_SHARED=0 \
     -DENABLE_UTILS=0 ${EMBEDDED_COMPILER} ${UNWIND} \
-    -DENABLE_ICU=0 -DENABLE_JEMALLOC=0 \
+    -DENABLE_ICU=0 ${JEMALLOC} \
     -DENABLE_PARQUET=1 -DENABLE_ROCKSDB=1 -DENABLE_SQLITE=1 -DENABLE_VECTORSCAN=1 \
     -DENABLE_PROTOBUF=1 -DENABLE_THRIFT=1 \
     -DENABLE_CLICKHOUSE_ALL=0 -DUSE_STATIC_LIBRARIES=1 -DSPLIT_SHARED_LIBRARIES=0 \
@@ -98,6 +100,13 @@ LIBCHDB_CMD=$(grep 'clang++.*-o programs/clickhouse .*' build.log \
     | sed 's/ -Xlinker --no-undefined//g' \
      )
 
+if [ "$(uname)" == "Linux" ]; then
+    # remove src/CMakeFiles/clickhouse_malloc.dir/Common/stubFree.c.o
+    LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed 's/ src\/CMakeFiles\/clickhouse_malloc.dir\/Common\/stubFree.c.o//g')
+    # put -Wl,-wrap,malloc ... after -DUSE_JEMALLOC=1
+    LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed 's/ -DUSE_JEMALLOC=1/ -DUSE_JEMALLOC=1 -Wl,-wrap,malloc -Wl,-wrap,valloc -Wl,-wrap,pvalloc -Wl,-wrap,calloc -Wl,-wrap,realloc -Wl,-wrap,memalign -Wl,-wrap,aligned_alloc -Wl,-wrap,posix_memalign -Wl,-wrap,free/g')
+fi
+
 # save the command to a file for debug
 echo ${LIBCHDB_CMD} > libchdb_cmd.sh
 

contrib/jemalloc-cmake/include/jemalloc/jemalloc.h

@@ -10,7 +10,7 @@ extern "C" {
 #endif
 
 #include <jemalloc/jemalloc_defs.h>
-#include <jemalloc/jemalloc_rename.h>
+// #include <jemalloc/jemalloc_rename.h>
 #include <jemalloc/jemalloc_macros.h>
 #include <jemalloc/jemalloc_protos.h>
 #include <jemalloc/jemalloc_typedefs.h>

contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -6,21 +6,19 @@
  * public APIs to be prefixed.  This makes it possible, with some care, to use
  * multiple allocators simultaneously.
  */
-/* #undef JEMALLOC_PREFIX */
-/* #undef JEMALLOC_CPREFIX */
+#define JEMALLOC_PREFIX "je_"
+#define JEMALLOC_CPREFIX "JE_"
 
 /*
  * Define overrides for non-standard allocator-related functions if they are
  * present on the system.
  */
-#if !defined(USE_MUSL)
-    #define JEMALLOC_OVERRIDE___LIBC_CALLOC
-    #define JEMALLOC_OVERRIDE___LIBC_FREE
-    #define JEMALLOC_OVERRIDE___LIBC_MALLOC
-    #define JEMALLOC_OVERRIDE___LIBC_MEMALIGN
-    #define JEMALLOC_OVERRIDE___LIBC_REALLOC
-    #define JEMALLOC_OVERRIDE___LIBC_VALLOC
-#endif
+/* #undef JEMALLOC_OVERRIDE___LIBC_CALLOC */
+/* #undef JEMALLOC_OVERRIDE___LIBC_FREE */
+/* #undef JEMALLOC_OVERRIDE___LIBC_MALLOC */
+/* #undef JEMALLOC_OVERRIDE___LIBC_MEMALIGN */
+/* #undef JEMALLOC_OVERRIDE___LIBC_REALLOC */
+/* #undef JEMALLOC_OVERRIDE___LIBC_VALLOC */
 /* #undef JEMALLOC_OVERRIDE___POSIX_MEMALIGN */
 
 /*
@@ -129,7 +127,7 @@
  * Among other things, it must be possible to initialize a mutex without
  * triggering allocation in order for threaded allocation to be safe.
  */
-#define JEMALLOC_THREADED_INIT
+/* #undef JEMALLOC_THREADED_INIT */
 
 /*
  * Defined if the pthreads implementation defines
@@ -139,7 +137,8 @@
 /* #undef JEMALLOC_MUTEX_INIT_CB */
 
 /* Non-empty if the tls_model attribute is supported. */
-#define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec")))
+// #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec")))
+#define JEMALLOC_TLS_MODEL
 
 /*
  * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables
@@ -374,10 +373,10 @@
 #define LG_SIZEOF_INTMAX_T 3
 
 /* glibc malloc hooks (__malloc_hook, __realloc_hook, __free_hook). */
-#define JEMALLOC_GLIBC_MALLOC_HOOK
+/* #undef JEMALLOC_GLIBC_MALLOC_HOOK */
 
 /* glibc memalign hook. */
-#define JEMALLOC_GLIBC_MEMALIGN_HOOK
+/* #undef JEMALLOC_GLIBC_MEMALIGN_HOOK */
 
 /* pthread support */
 #define JEMALLOC_HAVE_PTHREAD
@@ -397,7 +396,7 @@
 /*
  * If defined, all the features necessary for background threads are present.
  */
-#define JEMALLOC_BACKGROUND_THREAD
+/* #undef JEMALLOC_BACKGROUND_THREAD */
 
 /*
  * If defined, jemalloc symbols are not exported (doesn't work when
@@ -409,7 +408,7 @@
 #define JEMALLOC_CONFIG_MALLOC_CONF "@JEMALLOC_CONFIG_MALLOC_CONF@"
 
 /* If defined, jemalloc takes the malloc/free/etc. symbol names. */
-#define JEMALLOC_IS_MALLOC
+/* #undef JEMALLOC_IS_MALLOC */
 
 /*
  * Defined if strerror_r returns char * if _GNU_SOURCE is defined.

programs/local/LocalChdb.cpp

@@ -8,7 +8,7 @@
 
 namespace py = pybind11;
 
-// extern bool inside_main = true;
+extern bool inside_main = true;
 
 class __attribute__((visibility("default"))) query_result
 {

programs/main.cpp

@@ -344,6 +344,23 @@ struct Checker
 #endif
 ;
 
+// // if ENABLE_JEMALLOC is 1, we need to initialize it before any other memory allocation
+// // #if ENABLE_JEMALLOC
+// extern "C" {
+// extern bool malloc_init(void);
+// }
+// struct JEMallocInitializer
+// {
+//     JEMallocInitializer()
+//     {
+//         malloc_init();
+//         if (void * ptr = malloc(0))
+//         {
+//             free(ptr);
+//         }
+//     }
+// } jemalloc_initializer __attribute__((init_priority(101)));
+// // #endif
 
 #if !defined(DISABLE_HARMFUL_ENV_VAR_CHECK) && !defined(USE_MUSL)
 /// NOTE: We will migrate to full static linking or our own dynamic loader to make this code obsolete.
@@ -439,17 +456,17 @@ void checkHarmfulEnvironmentVariables(char ** argv)
 ///
 /// extern bool inside_main;
 /// class C { C() { assert(inside_main); } };
-#ifndef FUZZING_MODE
-bool inside_main = false;
-#else
-bool inside_main = true;
-#endif
+// #ifndef FUZZING_MODE
+// bool inside_main = false;
+// #else
+// bool inside_main = true;
+// #endif
 
 #if !defined(FUZZING_MODE)
 int main(int argc_, char ** argv_)
 {
-    inside_main = true;
-    SCOPE_EXIT({ inside_main = false; });
+    // inside_main = true;
+    // SCOPE_EXIT({ inside_main = false; });
 
     /// PHDR cache is required for query profiler to work reliably
     /// It also speed up exception handling, but exceptions from dynamically loaded libraries (dlopen)

programs/server/Server.cpp

@@ -148,7 +148,7 @@ static bool jemallocOptionEnabled(const char *name)
     bool value;
     size_t size = sizeof(value);
 
-    if (mallctl(name, reinterpret_cast<void *>(&value), &size, /* newp= */ nullptr, /* newlen= */ 0))
+    if (je_mallctl(name, reinterpret_cast<void *>(&value), &size, /* newp= */ nullptr, /* newlen= */ 0))
         throw Poco::SystemException("mallctl() failed");
 
     return value;

src/CMakeLists.txt

@@ -89,7 +89,7 @@ add_headers_and_sources(clickhouse_common_io Common/HashTable)
 add_headers_and_sources(clickhouse_common_io IO)
 add_headers_and_sources(clickhouse_common_io IO/Archives)
 add_headers_and_sources(clickhouse_common_io IO/S3)
-list (REMOVE_ITEM clickhouse_common_io_sources Common/malloc.cpp Common/new_delete.cpp)
+list (REMOVE_ITEM clickhouse_common_io_sources Common/malloc.cpp Common/mallocAdapt.c Common/stubFree.c Common/new_delete.cpp)
 
 add_headers_and_sources(dbms Disks/IO)
 add_headers_and_sources(dbms Disks/ObjectStorages)
@@ -202,7 +202,18 @@ if (SPLIT_SHARED_LIBRARIES)
     target_compile_definitions(clickhouse_common_io PRIVATE SPLIT_SHARED_LIBRARIES)
 endif ()
 
-add_library (clickhouse_malloc OBJECT Common/malloc.cpp)
+if (ENABLE_JEMALLOC)
+    add_library (clickhouse_malloc OBJECT Common/mallocAdapt.c Common/stubFree.c Common/malloc.cpp)
+    set_source_files_properties(Common/mallocAdapt.c PROPERTIES COMPILE_FLAGS "-DJEMALLOC_NO_RENAME")
+    set_source_files_properties(Common/mallocAdapt.c PROPERTIES COMPILE_FLAGS "-Wno-reserved-identifier") # suppress warning about __wrap_ functions
+    set_source_files_properties(Common/stubFree.c PROPERTIES COMPILE_FLAGS "-Wno-reserved-identifier") # suppress warning about __wrap_ functions
+    if (TARGET ch_contrib::jemalloc)
+        target_link_libraries (clickhouse_malloc PRIVATE ch_contrib::jemalloc)
+    endif()
+else()
+    add_library (clickhouse_malloc OBJECT Common/malloc.cpp)
+endif()
+
 set_source_files_properties(Common/malloc.cpp PROPERTIES COMPILE_FLAGS "-fno-builtin")
 
 if (((SANITIZE STREQUAL "thread") OR (SANITIZE STREQUAL "address")) AND COMPILER_GCC)
@@ -216,7 +227,7 @@ else()
 endif()
 
 if (TARGET ch_contrib::jemalloc)
-    target_link_libraries (clickhouse_common_io PRIVATE ch_contrib::jemalloc)
+    target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::jemalloc)
 endif()
 
 add_subdirectory(Access/Common)

src/Common/Allocator.h

@@ -11,8 +11,12 @@
 #include <pcg_random.hpp>
 #include <Common/thread_local_rng.h>
 
-#if !defined(OS_DARWIN) && !defined(OS_FREEBSD)
-#include <malloc.h>
+#if defined(USE_JEMALLOC) && (USE_JEMALLOC)
+#    include <jemalloc/jemalloc.h>
+#else
+#    if !defined(OS_DARWIN) && !defined(OS_FREEBSD)
+#        include <malloc.h>
+#    endif
 #endif
 
 #include <cstdlib>
@@ -130,8 +134,11 @@ class Allocator
         {
             /// Resize malloc'd memory region with no special alignment requirement.
             CurrentMemoryTracker::realloc(old_size, new_size);
-
+#if defined(USE_JEMALLOC) && (USE_JEMALLOC)
+            void * new_buf = je_realloc(buf, new_size);
+#else
             void * new_buf = ::realloc(buf, new_size);
+#endif
             if (nullptr == new_buf)
                 DB::throwFromErrno(fmt::format("Allocator: Cannot realloc from {} to {}.", ReadableSize(old_size), ReadableSize(new_size)), DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
 
@@ -221,11 +228,17 @@ class Allocator
         {
             if (alignment <= MALLOC_MIN_ALIGNMENT)
             {
+#if defined(USE_JEMALLOC) && (USE_JEMALLOC)
+                if constexpr (clear_memory)
+                    buf = je_calloc(size, 1);
+                else
+                    buf = je_malloc(size);
+#else
                 if constexpr (clear_memory)
                     buf = ::calloc(size, 1);
                 else
                     buf = ::malloc(size);
-
+#endif
                 if (nullptr == buf)
                     DB::throwFromErrno(fmt::format("Allocator: Cannot malloc {}.", ReadableSize(size)), DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY);
             }
@@ -254,7 +267,11 @@ class Allocator
         }
         else
         {
+#if defined(USE_JEMALLOC) && (USE_JEMALLOC)
+            je_free(buf);
+#else
             ::free(buf);
+#endif
         }
     }
 

src/Common/AsynchronousMetrics.cpp

@@ -349,7 +349,7 @@ uint64_t updateJemallocEpoch()
 {
     uint64_t value = 0;
     size_t size = sizeof(value);
-    mallctl("epoch", &value, &size, &value, size);
+    je_mallctl("epoch", &value, &size, &value, size);
     return value;
 }
 
@@ -361,7 +361,7 @@ static Value saveJemallocMetricImpl(
 {
     Value value{};
     size_t size = sizeof(value);
-    mallctl(jemalloc_full_name.c_str(), &value, &size, nullptr, 0);
+    je_mallctl(jemalloc_full_name.c_str(), &value, &size, nullptr, 0);
     values[clickhouse_full_name] = AsynchronousMetricValue(value, "An internal metric of the low-level memory allocator (jemalloc). See https://jemalloc.net/jemalloc.3.html");
     return value;
 }

src/Common/MemoryTracker.cpp

@@ -233,7 +233,7 @@ void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded, MemoryT
             if (free_memory_in_allocator_arenas.exchange(-current_free_memory_in_allocator_arenas) > 0)
             {
                 Stopwatch watch;
-                mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", nullptr, nullptr, nullptr, 0);
+                je_mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".purge", nullptr, nullptr, nullptr, 0);
                 ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurge);
                 ProfileEvents::increment(ProfileEvents::MemoryAllocatorPurgeTimeMicroseconds, watch.elapsedMicroseconds());
             }