Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

introduce jemalloc into thirdparty #6238

Merged
merged 2 commits into from Jul 20, 2022
Merged

introduce jemalloc into thirdparty #6238

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0d0eb1a
introduce jemalloc into thirdparty
silverbullet233 May 18, 2022
4b3f533
ensure forward compatibility
silverbullet233 Jul 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
7 changes: 4 additions & 3 deletions be/CMakeLists.txt
View file
Open in desktop
Expand Up @@ -213,8 +213,8 @@ set_target_properties(openssl PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib
add_library(leveldb STATIC IMPORTED)
set_target_properties(leveldb PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libleveldb.a)

add_library(jemalloc STATIC IMPORTED)
set_target_properties(jemalloc PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libjemalloc.a)
add_library(jemalloc_for_arrow STATIC IMPORTED)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why not consider use a --use-jemalloc to optionally use jemalloc ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why not consider use a --use-jemalloc to optionally use jemalloc ?

I will do this, but not in this PR…there are a lot of things to adapt to using jemalloc, such as mem_hook and some mertics

set_target_properties(jemalloc_for_arrow PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libjemalloc.a)

add_library(brotlicommon STATIC IMPORTED)
set_target_properties(brotlicommon PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libbrotlicommon.a)
Expand Down Expand Up @@ -562,7 +562,8 @@ set(STARROCKS_DEPENDENCIES
leveldb
bitshuffle
roaring
jemalloc
# arrow compiles jemalloc with a specific prefix, we should keep it
jemalloc_for_arrow
brotlicommon
brotlidec
brotlienc
Expand Down
16 changes: 16 additions & 0 deletions thirdparty/build-thirdparty.sh
View file
Open in desktop
Expand Up @@ -884,6 +884,21 @@ build_opentelemetry() {
${BUILD_SYSTEM} install
}

# jemalloc
build_jemalloc() {
OLD_CFLAGS=$CFLAGS
check_if_source_exist $JEMALLOC_SOURCE

unset CFLAGS
export CFLAGS="-O3 -fno-omit-frame-pointer -fPIC -g"
cd $TP_SOURCE_DIR/$JEMALLOC_SOURCE
./configure --prefix=${TP_INSTALL_DIR} --with-jemalloc-prefix=je --enable-prof --disable-cxx --disable-libdl --disable-shared
make -j$PARALLEL
make install
mv $TP_INSTALL_DIR/lib/libjemalloc.a $TP_INSTALL_DIR/lib/libjemalloc_for_starrocks.a
export CFLAGS=$OLD_CFLAGS
}

export CXXFLAGS="-O3 -fno-omit-frame-pointer -Wno-class-memaccess -fPIC -g -I${TP_INCLUDE_DIR}"
export CPPFLAGS=$CXXFLAGS
# https://stackoverflow.com/questions/42597685/storage-size-of-timespec-isnt-known
Expand Down Expand Up @@ -927,6 +942,7 @@ build_aliyun_oss_jars
build_aws_cpp_sdk
build_vpack
build_opentelemetry
build_jemalloc

if [[ "${MACHINE_TYPE}" != "aarch64" ]]; then
build_breakpad
Expand Down
17 changes: 17 additions & 0 deletions thirdparty/download-thirdparty.sh
View file
Open in desktop
Expand Up @@ -174,6 +174,7 @@ UNZIP_CMD="unzip"
SUFFIX_TGZ="\.(tar\.gz|tgz)$"
SUFFIX_XZ="\.tar\.xz$"
SUFFIX_ZIP="\.zip$"
SUFFIX_BZ2="\.bz2$"
# temporary directory for unpacking
# package is unpacked in tmp_dir and then renamed.
mkdir -p $TP_SOURCE_DIR/tmp_dir
Expand Down Expand Up @@ -206,6 +207,13 @@ do
echo "Failed to unzip ${!NAME}"
exit 1
fi
elif [[ "${!NAME}" =~ $SUFFIX_BZ2 ]]; then
echo "$TP_SOURCE_DIR/${!NAME}"
echo "$TP_SOURCE_DIR/${!SOURCE}"
if ! $TAR_CMD jxvf "$TP_SOURCE_DIR/${!NAME}" -C $TP_SOURCE_DIR/tmp_dir; then
echo "Failed to untar ${!NAME}"
exit 1
fi
else
echo "nothing has been done with ${!NAME}"
continue
Expand Down Expand Up @@ -375,5 +383,14 @@ else
echo "$AWS_SDK_CPP_SOURCE not patched"
fi

# patch jemalloc_hook
cd $TP_SOURCE_DIR/$JEMALLOC_SOURCE
if [ ! -f $PATCHED_MARK ] && [ $JEMALLOC_SOURCE = "jemalloc-5.2.1" ]; then
patch -p0 < $TP_PATCH_DIR/jemalloc_hook.patch
touch $PATCHED_MARK
fi
cd -
echo "Finished patching $JEMALLOC_SOURCE"

cd -

11 changes: 11 additions & 0 deletions thirdparty/patches/jemalloc_hook.patch
View file
Open in desktop
@@ -0,0 +1,11 @@
--- include/jemalloc/jemalloc.sh 2022-05-16 18:45:40.072099757 +0800
+++ include/jemalloc/jemalloc.sh 2022-05-16 18:45:44.994620722 +0800
@@ -12,7 +12,7 @@ extern "C" {
EOF

for hdr in jemalloc_defs.h jemalloc_rename.h jemalloc_macros.h \
- jemalloc_protos.h jemalloc_typedefs.h jemalloc_mangle.h ; do
+ jemalloc_protos.h jemalloc_typedefs.h ; do
cat "${objroot}include/jemalloc/${hdr}" \
| grep -v 'Generated from .* by configure\.' \
| sed -e 's/ $//g'