diff --git a/containers/buildUbuntuDevel.docker/Dockerfile.arm64 b/containers/buildUbuntuDevel.docker/Dockerfile.arm64 index 8d7500c30..fa5310160 100644 --- a/containers/buildUbuntuDevel.docker/Dockerfile.arm64 +++ b/containers/buildUbuntuDevel.docker/Dockerfile.arm64 @@ -1,4 +1,7 @@ # First rebuild glibc with a special option and provide some .deb files: +ARG COMPILER_VERSION=13 +ARG CLANG_VERSION=16 +ARG CLANG_FULL_VERSION=16.0.6 FROM ubuntu:24.04 as glibcdebs @@ -30,16 +33,61 @@ RUN dpkg -i /debs/*.deb # Finally, build the actual build image based on the above preparation: +# Build OpenMP static library +FROM ubuntu:24.04 as openmp + +ARG COMPILER_VERSION +ARG CLANG_VERSION +ARG CLANG_FULL_VERSION + +RUN apt-get update --fix-missing && \ + apt-get upgrade -y && \ + apt-get install -y software-properties-common net-tools wget + +WORKDIR / + +COPY static/ubuntu.sources ./ + +RUN cat ubuntu.sources >> /etc/apt/sources.list.d/ubuntu.sources + +RUN apt-get update && apt-get install dpkg-dev -y + +RUN apt-get source llvm-${CLANG_VERSION}-dev -y && \ + apt-get build-dep llvm-${CLANG_VERSION}-dev -y + +RUN apt-get update --fix-missing && apt-get install -y \ + clang-$CLANG_VERSION clang++-$CLANG_VERSION \ + cmake lsb-release wget git ninja-build gnupg + +COPY patches/diff_llvm.patch . + +WORKDIR /llvm-toolchain-${CLANG_VERSION}-${CLANG_FULL_VERSION} + +# Patch omp to enable running when it is a static library +RUN git apply /diff_llvm.patch + +WORKDIR build + +RUN cmake -G Ninja -DLLVM_ENABLE_PROJECTS="clang;openmp" -DCMAKE_C_COMPILER=clang-${CLANG_VERSION} \ + -DCMAKE_CXX_COMPILER=clang++-${CLANG_VERSION} -DCMAKE_BUILD_TYPE=RelWithDebInfo -DLIBOMP_ENABLE_SHARED=OFF \ + -DENABLE_OMPT_TOOLS=OFF -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -DCMAKE_INSTALL_PREFIX=/usr \ + -DENABLE_LIBOMPTARGET=OFF -DLIBOMP_INSTALL_ALIASES=OFF ../llvm && ninja + +RUN mkdir -p /opt/omp && cp ./lib/libomp.so.5 /opt/omp/libomp.a + + FROM prepare MAINTAINER Max Neunhoeffer ARG ARCH="aarch64" +ARG COMPILER_VERSION +ARG CLANG_VERSION # We provide gcc 13.2.0 as well as clang 16.0.6 from Ubunutu 24.04 -ENV COMPILER_VERSION 13 -ENV CLANG_VERSION 16 +ENV COMPILER_VERSION=${COMPILER_VERSION} +ENV CLANG_VERSION=${CLANG_VERSION} RUN apt-get update --fix-missing && \ apt-get install -y software-properties-common net-tools && \ @@ -48,7 +96,14 @@ RUN apt-get update --fix-missing && \ ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update --fix-missing && apt-get install -y build-essential gcc-${COMPILER_VERSION} g++-${COMPILER_VERSION} cmake make bison flex python3 ccache git libjemalloc-dev vim exuberant-ctags gdb fish psmisc sudo debhelper debconf jq wget libdb-dev curl gnupg2 gcovr prometheus bc tcpdump liburing-dev cppcheck clang-${CLANG_VERSION} lldb-${CLANG_VERSION} lld-${CLANG_VERSION} libc++-${CLANG_VERSION}-dev libc++abi-${CLANG_VERSION}-dev libclang-common-${CLANG_VERSION}-dev libclang-rt-${CLANG_VERSION}-dev nodejs npm && apt-get clean +RUN apt-get update --fix-missing && \ + apt-get install -y build-essential gcc-${COMPILER_VERSION} g++-${COMPILER_VERSION} \ + cmake make bison flex python3 ccache git libjemalloc-dev vim exuberant-ctags gdb fish \ + psmisc sudo debhelper debconf jq wget libdb-dev curl gnupg2 gcovr prometheus bc tcpdump \ + liburing-dev cppcheck clang-${CLANG_VERSION} lldb-${CLANG_VERSION} lld-${CLANG_VERSION} \ + libc++-${CLANG_VERSION}-dev libc++abi-${CLANG_VERSION}-dev libclang-common-${CLANG_VERSION}-dev \ + libclang-rt-${CLANG_VERSION}-dev nodejs npm libomp-16-dev liblapack-dev gfortran \ + && apt-get clean # yarn for UI build: RUN npm install -g yarn @@ -61,6 +116,7 @@ RUN curl -L https://github.com/mozilla/sccache/releases/download/v0.3.1/sccache- #COPY --from=v8 /opt/v8/lib /opt/v8/lib #COPY --from=v8 /opt/v8/include /opt/v8/include #COPY --from=v8 /opt/v8/third_party /opt/v8/third_party +COPY --from=openmp /opt/omp /opt/omp # Build openssl and openldap libraries in specific versions and with static # libraries, install with prefix /opt: diff --git a/containers/buildUbuntuDevel.docker/Dockerfile.x86-64 b/containers/buildUbuntuDevel.docker/Dockerfile.x86-64 index 57f2be3af..3d4294ee7 100644 --- a/containers/buildUbuntuDevel.docker/Dockerfile.x86-64 +++ b/containers/buildUbuntuDevel.docker/Dockerfile.x86-64 @@ -1,5 +1,9 @@ -# First rebuild glibc with a special option and provide some .deb files: +# Arguments used through multiple build stages +ARG COMPILER_VERSION=13 +ARG CLANG_VERSION=16 +ARG CLANG_FULL_VERSION=16.0.6 +# First rebuild glibc with a special option and provide some .deb files: FROM ubuntu:24.04 as glibcdebs COPY ./static /static @@ -28,6 +32,49 @@ RUN dpkg -i /debs/*.deb #COPY ./tools/install_v8.sh /tools/install_v8.sh #RUN /tools/install_v8.sh +# Build OpenMP static library +FROM ubuntu:24.04 as openmp + +ARG COMPILER_VERSION +ARG CLANG_VERSION +ARG CLANG_FULL_VERSION + +RUN apt-get update --fix-missing && \ + apt-get upgrade -y && \ + apt-get install -y software-properties-common net-tools wget + +WORKDIR / + +COPY static/ubuntu.sources ./ + +RUN cat ubuntu.sources >> /etc/apt/sources.list.d/ubuntu.sources + +RUN apt-get update && apt-get install dpkg-dev -y + +RUN apt-get source llvm-${CLANG_VERSION}-dev -y && \ + apt-get build-dep llvm-${CLANG_VERSION}-dev -y + +RUN apt-get update --fix-missing && apt-get install -y \ + clang-$CLANG_VERSION clang++-$CLANG_VERSION \ + cmake lsb-release wget git ninja-build gnupg + +COPY patches/diff_llvm.patch . + +WORKDIR /llvm-toolchain-${CLANG_VERSION}-${CLANG_FULL_VERSION} + +# Patch omp to enable running when it is a static library +RUN git apply /diff_llvm.patch + +WORKDIR build + +RUN cmake -G Ninja -DLLVM_ENABLE_PROJECTS="clang;openmp" -DCMAKE_C_COMPILER=clang-${CLANG_VERSION} \ + -DCMAKE_CXX_COMPILER=clang++-${CLANG_VERSION} -DCMAKE_BUILD_TYPE=RelWithDebInfo -DLIBOMP_ENABLE_SHARED=OFF \ + -DENABLE_OMPT_TOOLS=OFF -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -DCMAKE_INSTALL_PREFIX=/usr \ + -DENABLE_LIBOMPTARGET=OFF -DLIBOMP_INSTALL_ALIASES=OFF ../llvm && ninja + +RUN mkdir -p /opt/omp && cp ./lib/libomp.so.5 /opt/omp/libomp.a + + # Finally, build the actual build image based on the above preparation: FROM prepare @@ -35,11 +82,13 @@ FROM prepare MAINTAINER Max Neunhoeffer ARG ARCH="x86_64" +ARG COMPILER_VERSION +ARG CLANG_VERSION # We provide gcc 13.2.0 as well as clang 16.0.6 from Ubunutu 24.04 -ENV COMPILER_VERSION 13 -ENV CLANG_VERSION 16 +ENV COMPILER_VERSION=${COMPILER_VERSION} +ENV CLANG_VERSION=${CLANG_VERSION} RUN apt-get update --fix-missing && \ apt-get install -y software-properties-common net-tools && \ @@ -48,7 +97,14 @@ RUN apt-get update --fix-missing && \ ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update --fix-missing && apt-get install -y build-essential gcc-${COMPILER_VERSION} g++-${COMPILER_VERSION} cmake make bison flex python3 ccache git libjemalloc-dev vim exuberant-ctags gdb fish psmisc sudo debhelper debconf jq wget libdb-dev curl gnupg2 gcovr prometheus bc tcpdump liburing-dev cppcheck clang-${CLANG_VERSION} lldb-${CLANG_VERSION} lld-${CLANG_VERSION} libc++-${CLANG_VERSION}-dev libc++abi-${CLANG_VERSION}-dev libclang-common-${CLANG_VERSION}-dev libclang-rt-${CLANG_VERSION}-dev nodejs npm && apt-get clean +RUN apt-get update --fix-missing && \ + apt-get install -y build-essential gcc-${COMPILER_VERSION} g++-${COMPILER_VERSION} \ + cmake make bison flex python3 ccache git libjemalloc-dev vim exuberant-ctags gdb fish \ + psmisc sudo debhelper debconf jq wget libdb-dev curl gnupg2 gcovr prometheus bc tcpdump \ + liburing-dev cppcheck clang-${CLANG_VERSION} lldb-${CLANG_VERSION} lld-${CLANG_VERSION} \ + libc++-${CLANG_VERSION}-dev libc++abi-${CLANG_VERSION}-dev libclang-common-${CLANG_VERSION}-dev \ + libclang-rt-${CLANG_VERSION}-dev nodejs npm libomp-16-dev liblapack-dev gfortran \ + && apt-get clean # yarn for UI build: RUN npm install -g yarn @@ -62,6 +118,9 @@ RUN curl -L https://github.com/mozilla/sccache/releases/download/v0.3.1/sccache- #COPY --from=v8 /opt/v8/include /opt/v8/include #COPY --from=v8 /opt/v8/third_party /opt/v8/third_party +# Fetch libomp +COPY --from=openmp /opt/omp /opt/omp + # Build openssl and openldap libraries in specific versions and with static # libraries, install with prefix /opt: diff --git a/containers/buildUbuntuDevel.docker/patches/diff_llvm.patch b/containers/buildUbuntuDevel.docker/patches/diff_llvm.patch new file mode 100644 index 000000000..632c2f10c --- /dev/null +++ b/containers/buildUbuntuDevel.docker/patches/diff_llvm.patch @@ -0,0 +1,86 @@ +diff --git a/openmp/runtime/src/ompd-specific.cpp b/openmp/runtime/src/ompd-specific.cpp +index c4018789e..07da27023 100644 +--- a/openmp/runtime/src/ompd-specific.cpp ++++ b/openmp/runtime/src/ompd-specific.cpp +@@ -82,6 +82,7 @@ void ompd_init() { + char *libname = NULL; + + #if KMP_OS_UNIX ++#if 0 + // Find the location of libomp.so thru dladdr and replace the libomp with + // libompd to get the full path of libompd + Dl_info dl_info; +@@ -97,6 +98,7 @@ void ompd_init() { + strncpy(libname, dl_info.dli_fname, lib_path_length); + memcpy(libname + lib_path_length, "/libompd.so\0", 12); + } ++#endif + #endif + + const char *ompd_env_var = getenv("OMP_DEBUG"); +diff --git a/openmp/runtime/src/ompt-general.cpp b/openmp/runtime/src/ompt-general.cpp +index 55a936575..a71c1aef3 100644 +--- a/openmp/runtime/src/ompt-general.cpp ++++ b/openmp/runtime/src/ompt-general.cpp +@@ -164,8 +164,7 @@ ompt_start_tool(unsigned int omp_version, const char *runtime_version) { + // runtime library is linked before the tool. Since glibc 2.2 strong symbols + // don't override weak symbols that have been found before unless the user + // sets the environment variable LD_DYNAMIC_WEAK. +- ompt_start_tool_t next_tool = +- (ompt_start_tool_t)dlsym(RTLD_NEXT, "ompt_start_tool"); ++ ompt_start_tool_t next_tool = nullptr; + if (next_tool) { + ret = next_tool(omp_version, runtime_version); + } +@@ -245,14 +244,14 @@ ompt_tool_windows(unsigned int omp_version, const char *runtime_version) { + #endif + + static ompt_start_tool_result_t * +-ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) { ++ ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) { + ompt_start_tool_result_t *ret = NULL; +- ompt_start_tool_t start_tool = NULL; ++ // ompt_start_tool_t start_tool = NULL; + #if KMP_OS_WINDOWS + // Cannot use colon to describe a list of absolute paths on Windows + const char *sep = ";"; + #else +- const char *sep = ":"; ++ // const char *sep = ":"; + #endif + + OMPT_VERBOSE_INIT_PRINT("----- START LOGGING OF TOOL REGISTRATION -----\n"); +@@ -275,7 +274,7 @@ ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) { + OMPT_VERBOSE_INIT_PRINT("----- END LOGGING OF TOOL REGISTRATION -----\n"); + return ret; + } +- ++#if 0 + // Try tool-libraries-var ICV + OMPT_VERBOSE_INIT_CONTINUED_PRINT("Failed.\n"); + const char *tool_libs = getenv("OMP_TOOL_LIBRARIES"); +@@ -348,6 +347,8 @@ ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) { + } else { + OMPT_VERBOSE_INIT_PRINT("No OMP_TOOL_LIBRARIES defined.\n"); + } ++#endif ++ return nullptr; + + // usable tool found in tool-libraries + if (ret) { +@@ -355,6 +356,7 @@ ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) { + return ret; + } + ++#if 0 + #if KMP_OS_UNIX + { // Non-standard: load archer tool if application is built with TSan + const char *fname = "libarcher.so"; +@@ -383,6 +385,7 @@ ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) { + } + } + } ++#endif + #endif + OMPT_VERBOSE_INIT_PRINT("No OMP tool loaded.\n"); + OMPT_VERBOSE_INIT_PRINT("----- END LOGGING OF TOOL REGISTRATION -----\n"); diff --git a/containers/buildUbuntuDevel.docker/static/ubuntu.sources b/containers/buildUbuntuDevel.docker/static/ubuntu.sources new file mode 100644 index 000000000..12e4ee8e0 --- /dev/null +++ b/containers/buildUbuntuDevel.docker/static/ubuntu.sources @@ -0,0 +1,15 @@ + +Types: deb-src +URIs: http://archive.ubuntu.com/ubuntu/ +Suites: noble noble-updates noble-backports +Components: main universe restricted multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + +## Ubuntu security updates. Aside from URIs and Suites, +## this should mirror your choices in the previous section. +Types: deb-src +URIs: http://security.ubuntu.com/ubuntu/ +Suites: noble-security +Components: main universe restricted multiverse +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg + diff --git a/helper.linux.fish b/helper.linux.fish index d073c7a2f..681d922aa 100755 --- a/helper.linux.fish +++ b/helper.linux.fish @@ -22,7 +22,7 @@ else end set -gx UBUNTUBUILDIMAGE_312_NAME arangodb/ubuntubuildarangodb-devel -set -gx UBUNTUBUILDIMAGE_312_TAG 7 +set -gx UBUNTUBUILDIMAGE_312_TAG 8 set -gx UBUNTUBUILDIMAGE_312 $UBUNTUBUILDIMAGE_312_NAME:$UBUNTUBUILDIMAGE_312_TAG-$UBUNTUBUILDIMAGE_TAG_ARCH set -gx UBUNTUBUILDIMAGE_311_NAME $UBUNTUBUILDIMAGE_312_NAME diff --git a/scripts/buildArangoDB312.fish b/scripts/buildArangoDB312.fish index b516ac804..faa578042 100755 --- a/scripts/buildArangoDB312.fish +++ b/scripts/buildArangoDB312.fish @@ -37,6 +37,12 @@ if test "$ARCH" = "x86_64" -a (string sub -s 1 -l 1 "$OPENSSLPATH") = "3" set -xg X86_64_SUFFIX "64" end +if test "$ARCH" = "x86_64" + set -xg LAPACK_LIB_PATH "/usr/lib/x86_64-linux-gnu/lapack/liblapack.a" +else + set -xg LAPACK_LIB_PATH "/usr/lib/aarch64-linux-gnu/lapack/liblapack.a" +end + set -l pie "" if test "$STATIC_EXECUTABLES" = "" @@ -53,15 +59,16 @@ set -g FULLARGS $argv \ -DOPENSSL_ROOT_DIR=/opt \ -DUSE_STRICT_OPENSSL_VERSION=$USE_STRICT_OPENSSL \ -DBUILD_REPO_INFO=$BUILD_REPO_INFO \ - -DARANGODB_BUILD_DATE="$ARANGODB_BUILD_DATE" + -DARANGODB_BUILD_DATE="$ARANGODB_BUILD_DATE" \ + -DLAPACK_LIBRARIES="$LAPACK_LIB_PATH" if test "$MAINTAINER" = "On" set -g FULLARGS $FULLARGS \ - -DCMAKE_EXE_LINKER_FLAGS="-Wl,--build-id=sha1 $pie -fno-stack-protector -fuse-ld=lld" \ + -DCMAKE_EXE_LINKER_FLAGS="-Wl,--build-id=sha1 $pie -fno-stack-protector -fuse-ld=lld -fopenmp=libomp -L/opt/omp -llapack -lgfortran" \ -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld" else set -g FULLARGS $FULLARGS \ - -DCMAKE_EXE_LINKER_FLAGS="-Wl,--build-id=sha1 $pie $inline -fno-stack-protector -fuse-ld=lld " \ + -DCMAKE_EXE_LINKER_FLAGS="-Wl,--build-id=sha1 $pie $inline -fno-stack-protector -fuse-ld=lld -fopenmp=libomp -L/opt/omp -llapack -lgfortran" \ -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld" \ -DUSE_CATCH_TESTS=Off \ -DUSE_GOOGLE_TESTS=Off