introduce jemalloc into thirdparty

StarRocks · Jul 18, 2022 · 0d0eb1a · 0d0eb1a
1 parent 2102bec
commit 0d0eb1a
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 4 deletions.
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
@@ -213,8 +213,8 @@ set_target_properties(openssl PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib
 add_library(leveldb STATIC IMPORTED)
 set_target_properties(leveldb PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libleveldb.a)
 
-add_library(jemalloc STATIC IMPORTED)
-set_target_properties(jemalloc PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libjemalloc.a)
+add_library(jemalloc_for_arrow STATIC IMPORTED)
+set_target_properties(jemalloc_for_arrow PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libjemalloc_for_arrow.a)
 
 add_library(brotlicommon STATIC IMPORTED)
 set_target_properties(brotlicommon PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libbrotlicommon.a)
@@ -562,7 +562,8 @@ set(STARROCKS_DEPENDENCIES
     leveldb
     bitshuffle
     roaring
-    jemalloc
+    # arrow compiles jemalloc with a specific prefix, we should keep it
+    jemalloc_for_arrow
     brotlicommon
     brotlidec
     brotlienc

diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
@@ -564,7 +564,7 @@ build_arrow() {
     ${BUILD_SYSTEM} -j$PARALLEL
     ${BUILD_SYSTEM} install
     #copy dep libs
-    cp -rf ./jemalloc_ep-prefix/src/jemalloc_ep/dist/lib/libjemalloc_pic.a $TP_INSTALL_DIR/lib64/libjemalloc.a
+    cp -rf ./jemalloc_ep-prefix/src/jemalloc_ep/dist/lib/libjemalloc_pic.a $TP_INSTALL_DIR/lib64/libjemalloc_for_arrow.a
     cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlienc-static.a $TP_INSTALL_DIR/lib64/libbrotlienc.a
     cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlidec-static.a $TP_INSTALL_DIR/lib64/libbrotlidec.a
     cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlicommon-static.a $TP_INSTALL_DIR/lib64/libbrotlicommon.a
@@ -884,6 +884,20 @@ build_opentelemetry() {
     ${BUILD_SYSTEM} install
 }
 
+# jemalloc
+build_jemalloc() {
+    OLD_CFLAGS=$CFLAGS
+    check_if_source_exist $JEMALLOC_SOURCE
+
+    unset CFLAGS
+    export CFLAGS="-O3 -fno-omit-frame-pointer -fPIC -g"
+    cd $TP_SOURCE_DIR/$JEMALLOC_SOURCE
+    ./configure --prefix=${TP_INSTALL_DIR} --with-jemalloc-prefix=je --enable-prof --disable-cxx --disable-libdl --disable-shared
+    make -j$PARALLEL
+    make install
+    export CFLAGS=$OLD_CFLAGS
+}
+
 export CXXFLAGS="-O3 -fno-omit-frame-pointer -Wno-class-memaccess -fPIC -g -I${TP_INCLUDE_DIR}"
 export CPPFLAGS=$CXXFLAGS
 # https://stackoverflow.com/questions/42597685/storage-size-of-timespec-isnt-known
@@ -927,6 +941,7 @@ build_aliyun_oss_jars
 build_aws_cpp_sdk
 build_vpack
 build_opentelemetry
+build_jemalloc
 
 if [[ "${MACHINE_TYPE}" != "aarch64" ]]; then
     build_breakpad

diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh
@@ -375,5 +375,14 @@ else
     echo "$AWS_SDK_CPP_SOURCE not patched"
 fi
 
+# patch jemalloc_hook
+cd $TP_SOURCE_DIR/$JEMALLOC_SOURCE
+if [ ! -f $PATCHED_MARK ] && [ $JEMALLOC_SOURCE = "jemalloc-5.2.1" ]; then
+    patch -p0 < $TP_PATCH_DIR/jemalloc_hook.patch
+    touch $PATCHED_MARK
+fi
+cd -
+echo "Finished patching $JEMALLOC_SOURCE"
+
 cd -
 
diff --git a/thirdparty/patches/jemalloc_hook.patch b/thirdparty/patches/jemalloc_hook.patch
@@ -0,0 +1,11 @@
+--- include/jemalloc/jemalloc.sh	2022-05-16 18:45:40.072099757 +0800
++++ include/jemalloc/jemalloc.sh	2022-05-16 18:45:44.994620722 +0800
+@@ -12,7 +12,7 @@ extern "C" {
+ EOF
+
+ for hdr in jemalloc_defs.h jemalloc_rename.h jemalloc_macros.h \
+-           jemalloc_protos.h jemalloc_typedefs.h jemalloc_mangle.h ; do
++           jemalloc_protos.h jemalloc_typedefs.h ; do
+   cat "${objroot}include/jemalloc/${hdr}" \
+       | grep -v 'Generated from .* by configure\.' \
+       | sed -e 's/ $//g'