Support vector-index changes

containers/buildUbuntuDevel.docker/Dockerfile.arm64

@@ -1,4 +1,7 @@
 # First rebuild glibc with a special option and provide some .deb files:
+ARG COMPILER_VERSION=13
+ARG CLANG_VERSION=16
+ARG CLANG_FULL_VERSION=16.0.6
 
 FROM ubuntu:24.04 as glibcdebs
 
@@ -30,16 +33,61 @@ RUN dpkg -i /debs/*.deb
 
 # Finally, build the actual build image based on the above preparation:
 
+# Build OpenMP static library
+FROM ubuntu:24.04 as openmp
+
+ARG COMPILER_VERSION
+ARG CLANG_VERSION
+ARG CLANG_FULL_VERSION
+
+RUN apt-get update --fix-missing && \
+    apt-get upgrade -y && \
+    apt-get install -y software-properties-common net-tools wget
+
+WORKDIR /
+
+COPY static/ubuntu.sources ./
+
+RUN cat ubuntu.sources >> /etc/apt/sources.list.d/ubuntu.sources
+
+RUN apt-get update && apt-get install dpkg-dev -y
+
+RUN apt-get source llvm-${CLANG_VERSION}-dev -y && \
+  apt-get build-dep llvm-${CLANG_VERSION}-dev -y
+
+RUN apt-get update --fix-missing && apt-get install -y \
+  clang-$CLANG_VERSION clang++-$CLANG_VERSION \
+  cmake lsb-release wget git ninja-build gnupg
+
+COPY patches/diff_llvm.patch .
+
+WORKDIR /llvm-toolchain-${CLANG_VERSION}-${CLANG_FULL_VERSION}
+
+# Patch omp to enable running when it is a static library
+RUN git apply /diff_llvm.patch 
+
+WORKDIR build
+
+RUN cmake -G Ninja -DLLVM_ENABLE_PROJECTS="clang;openmp" -DCMAKE_C_COMPILER=clang-${CLANG_VERSION} \
+  -DCMAKE_CXX_COMPILER=clang++-${CLANG_VERSION} -DCMAKE_BUILD_TYPE=RelWithDebInfo -DLIBOMP_ENABLE_SHARED=OFF \
+  -DENABLE_OMPT_TOOLS=OFF -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -DCMAKE_INSTALL_PREFIX=/usr \
+  -DENABLE_LIBOMPTARGET=OFF -DLIBOMP_INSTALL_ALIASES=OFF ../llvm && ninja
+
+RUN mkdir -p /opt/omp && cp ./lib/libomp.so.5 /opt/omp/libomp.a
+
+
 FROM prepare
 
 MAINTAINER Max Neunhoeffer <max@arangodb.com>
 
 ARG ARCH="aarch64"
+ARG COMPILER_VERSION
+ARG CLANG_VERSION
 
 # We provide gcc 13.2.0 as well as clang 16.0.6 from Ubunutu 24.04
 
-ENV COMPILER_VERSION 13
-ENV CLANG_VERSION 16
+ENV COMPILER_VERSION=${COMPILER_VERSION}
+ENV CLANG_VERSION=${CLANG_VERSION}
 
 RUN apt-get update --fix-missing && \
     apt-get install -y software-properties-common net-tools && \
@@ -48,7 +96,14 @@ RUN apt-get update --fix-missing && \
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-RUN apt-get update --fix-missing && apt-get install -y build-essential gcc-${COMPILER_VERSION} g++-${COMPILER_VERSION} cmake make bison flex python3 ccache git libjemalloc-dev vim exuberant-ctags gdb fish psmisc sudo debhelper debconf jq wget libdb-dev curl gnupg2 gcovr prometheus bc tcpdump liburing-dev cppcheck clang-${CLANG_VERSION} lldb-${CLANG_VERSION} lld-${CLANG_VERSION} libc++-${CLANG_VERSION}-dev libc++abi-${CLANG_VERSION}-dev libclang-common-${CLANG_VERSION}-dev libclang-rt-${CLANG_VERSION}-dev nodejs npm && apt-get clean
+RUN apt-get update --fix-missing && \
+  apt-get install -y build-essential gcc-${COMPILER_VERSION} g++-${COMPILER_VERSION} \
+  cmake make bison flex python3 ccache git libjemalloc-dev vim exuberant-ctags gdb fish \
+  psmisc sudo debhelper debconf jq wget libdb-dev curl gnupg2 gcovr prometheus bc tcpdump \
+  liburing-dev cppcheck clang-${CLANG_VERSION} lldb-${CLANG_VERSION} lld-${CLANG_VERSION} \
+  libc++-${CLANG_VERSION}-dev libc++abi-${CLANG_VERSION}-dev libclang-common-${CLANG_VERSION}-dev \
+  libclang-rt-${CLANG_VERSION}-dev nodejs npm libomp-16-dev liblapack-dev gfortran \
+  && apt-get clean
 
 # yarn for UI build:
 RUN npm install -g yarn
@@ -61,6 +116,7 @@ RUN curl -L https://github.com/mozilla/sccache/releases/download/v0.3.1/sccache-
 #COPY --from=v8 /opt/v8/lib /opt/v8/lib
 #COPY --from=v8 /opt/v8/include /opt/v8/include
 #COPY --from=v8 /opt/v8/third_party /opt/v8/third_party
+COPY --from=openmp /opt/omp /opt/omp 
 
 # Build openssl and openldap libraries in specific versions and with static
 # libraries, install with prefix /opt:

containers/buildUbuntuDevel.docker/Dockerfile.x86-64

@@ -1,5 +1,9 @@
-# First rebuild glibc with a special option and provide some .deb files:
+# Arguments used through multiple build stages
+ARG COMPILER_VERSION=13
+ARG CLANG_VERSION=16
+ARG CLANG_FULL_VERSION=16.0.6
 
+# First rebuild glibc with a special option and provide some .deb files:
 FROM ubuntu:24.04 as glibcdebs
 
 COPY ./static /static
@@ -28,18 +32,63 @@ RUN dpkg -i /debs/*.deb
 #COPY ./tools/install_v8.sh /tools/install_v8.sh
 #RUN /tools/install_v8.sh
 
+# Build OpenMP static library
+FROM ubuntu:24.04 as openmp
+
+ARG COMPILER_VERSION
+ARG CLANG_VERSION
+ARG CLANG_FULL_VERSION
+
+RUN apt-get update --fix-missing && \
+    apt-get upgrade -y && \
+    apt-get install -y software-properties-common net-tools wget
+
+WORKDIR /
+
+COPY static/ubuntu.sources ./
+
+RUN cat ubuntu.sources >> /etc/apt/sources.list.d/ubuntu.sources
+
+RUN apt-get update && apt-get install dpkg-dev -y
+
+RUN apt-get source llvm-${CLANG_VERSION}-dev -y && \
+  apt-get build-dep llvm-${CLANG_VERSION}-dev -y
+
+RUN apt-get update --fix-missing && apt-get install -y \
+  clang-$CLANG_VERSION clang++-$CLANG_VERSION \
+  cmake lsb-release wget git ninja-build gnupg
+
+COPY patches/diff_llvm.patch .
+
+WORKDIR /llvm-toolchain-${CLANG_VERSION}-${CLANG_FULL_VERSION}
+
+# Patch omp to enable running when it is a static library
+RUN git apply /diff_llvm.patch 
+
+WORKDIR build
+
+RUN cmake -G Ninja -DLLVM_ENABLE_PROJECTS="clang;openmp" -DCMAKE_C_COMPILER=clang-${CLANG_VERSION} \
+  -DCMAKE_CXX_COMPILER=clang++-${CLANG_VERSION} -DCMAKE_BUILD_TYPE=RelWithDebInfo -DLIBOMP_ENABLE_SHARED=OFF \
+  -DENABLE_OMPT_TOOLS=OFF -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -DCMAKE_INSTALL_PREFIX=/usr \
+  -DENABLE_LIBOMPTARGET=OFF -DLIBOMP_INSTALL_ALIASES=OFF ../llvm && ninja
+
+RUN mkdir -p /opt/omp && cp ./lib/libomp.so.5 /opt/omp/libomp.a
+
+
 # Finally, build the actual build image based on the above preparation:
 
 FROM prepare
 
 MAINTAINER Max Neunhoeffer <max@arangodb.com>
 
 ARG ARCH="x86_64"
+ARG COMPILER_VERSION
+ARG CLANG_VERSION
 
 # We provide gcc 13.2.0 as well as clang 16.0.6 from Ubunutu 24.04
 
-ENV COMPILER_VERSION 13
-ENV CLANG_VERSION 16
+ENV COMPILER_VERSION=${COMPILER_VERSION}
+ENV CLANG_VERSION=${CLANG_VERSION}
 
 RUN apt-get update --fix-missing && \
     apt-get install -y software-properties-common net-tools && \
@@ -48,7 +97,14 @@ RUN apt-get update --fix-missing && \
 
 ENV DEBIAN_FRONTEND=noninteractive
 
-RUN apt-get update --fix-missing && apt-get install -y build-essential gcc-${COMPILER_VERSION} g++-${COMPILER_VERSION} cmake make bison flex python3 ccache git libjemalloc-dev vim exuberant-ctags gdb fish psmisc sudo debhelper debconf jq wget libdb-dev curl gnupg2 gcovr prometheus bc tcpdump liburing-dev cppcheck clang-${CLANG_VERSION} lldb-${CLANG_VERSION} lld-${CLANG_VERSION} libc++-${CLANG_VERSION}-dev libc++abi-${CLANG_VERSION}-dev libclang-common-${CLANG_VERSION}-dev libclang-rt-${CLANG_VERSION}-dev nodejs npm && apt-get clean
+RUN apt-get update --fix-missing && \
+  apt-get install -y build-essential gcc-${COMPILER_VERSION} g++-${COMPILER_VERSION} \
+  cmake make bison flex python3 ccache git libjemalloc-dev vim exuberant-ctags gdb fish \
+  psmisc sudo debhelper debconf jq wget libdb-dev curl gnupg2 gcovr prometheus bc tcpdump \
+  liburing-dev cppcheck clang-${CLANG_VERSION} lldb-${CLANG_VERSION} lld-${CLANG_VERSION} \
+  libc++-${CLANG_VERSION}-dev libc++abi-${CLANG_VERSION}-dev libclang-common-${CLANG_VERSION}-dev \
+  libclang-rt-${CLANG_VERSION}-dev nodejs npm libomp-16-dev liblapack-dev gfortran \
+  && apt-get clean
 
 # yarn for UI build:
 RUN npm install -g yarn
@@ -62,6 +118,9 @@ RUN curl -L https://github.com/mozilla/sccache/releases/download/v0.3.1/sccache-
 #COPY --from=v8 /opt/v8/include /opt/v8/include
 #COPY --from=v8 /opt/v8/third_party /opt/v8/third_party
 
+# Fetch libomp
+COPY --from=openmp /opt/omp /opt/omp 
+
 # Build openssl and openldap libraries in specific versions and with static
 # libraries, install with prefix /opt:
 

containers/buildUbuntuDevel.docker/patches/diff_llvm.patch

@@ -0,0 +1,86 @@
+diff --git a/openmp/runtime/src/ompd-specific.cpp b/openmp/runtime/src/ompd-specific.cpp
+index c4018789e..07da27023 100644
+--- a/openmp/runtime/src/ompd-specific.cpp
++++ b/openmp/runtime/src/ompd-specific.cpp
+@@ -82,6 +82,7 @@ void ompd_init() {
+   char *libname = NULL;
+
+ #if KMP_OS_UNIX
++#if 0
+   // Find the location of libomp.so thru dladdr and replace the libomp with
+   // libompd to get the full path of libompd
+   Dl_info dl_info;
+@@ -97,6 +98,7 @@ void ompd_init() {
+     strncpy(libname, dl_info.dli_fname, lib_path_length);
+     memcpy(libname + lib_path_length, "/libompd.so\0", 12);
+   }
++#endif
+ #endif
+
+   const char *ompd_env_var = getenv("OMP_DEBUG");
+diff --git a/openmp/runtime/src/ompt-general.cpp b/openmp/runtime/src/ompt-general.cpp
+index 55a936575..a71c1aef3 100644
+--- a/openmp/runtime/src/ompt-general.cpp
++++ b/openmp/runtime/src/ompt-general.cpp
+@@ -164,8 +164,7 @@ ompt_start_tool(unsigned int omp_version, const char *runtime_version) {
+   // runtime library is linked before the tool. Since glibc 2.2 strong symbols
+   // don't override weak symbols that have been found before unless the user
+   // sets the environment variable LD_DYNAMIC_WEAK.
+-  ompt_start_tool_t next_tool =
+-      (ompt_start_tool_t)dlsym(RTLD_NEXT, "ompt_start_tool");
++  ompt_start_tool_t next_tool = nullptr;
+   if (next_tool) {
+     ret = next_tool(omp_version, runtime_version);
+   }
+@@ -245,14 +244,14 @@ ompt_tool_windows(unsigned int omp_version, const char *runtime_version) {
+ #endif
+
+ static ompt_start_tool_result_t *
+-ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) {
++ ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) {
+   ompt_start_tool_result_t *ret = NULL;
+-  ompt_start_tool_t start_tool = NULL;
++  // ompt_start_tool_t start_tool = NULL;
+ #if KMP_OS_WINDOWS
+   // Cannot use colon to describe a list of absolute paths on Windows
+   const char *sep = ";";
+ #else
+-  const char *sep = ":";
++  // const char *sep = ":";
+ #endif
+
+   OMPT_VERBOSE_INIT_PRINT("----- START LOGGING OF TOOL REGISTRATION -----\n");
+@@ -275,7 +274,7 @@ ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) {
+     OMPT_VERBOSE_INIT_PRINT("----- END LOGGING OF TOOL REGISTRATION -----\n");
+     return ret;
+   }
+-
++#if 0
+   // Try tool-libraries-var ICV
+   OMPT_VERBOSE_INIT_CONTINUED_PRINT("Failed.\n");
+   const char *tool_libs = getenv("OMP_TOOL_LIBRARIES");
+@@ -348,6 +347,8 @@ ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) {
+   } else {
+     OMPT_VERBOSE_INIT_PRINT("No OMP_TOOL_LIBRARIES defined.\n");
+   }
++#endif
++  return nullptr;
+
+   // usable tool found in tool-libraries
+   if (ret) {
+@@ -355,6 +356,7 @@ ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) {
+     return ret;
+   }
+
++#if 0
+ #if KMP_OS_UNIX
+   { // Non-standard: load archer tool if application is built with TSan
+     const char *fname = "libarcher.so";
+@@ -383,6 +385,7 @@ ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) {
+       }
+     }
+   }
++#endif
+ #endif
+   OMPT_VERBOSE_INIT_PRINT("No OMP tool loaded.\n");
+   OMPT_VERBOSE_INIT_PRINT("----- END LOGGING OF TOOL REGISTRATION -----\n");

containers/buildUbuntuDevel.docker/static/ubuntu.sources

@@ -0,0 +1,15 @@
+
+Types: deb-src
+URIs: http://archive.ubuntu.com/ubuntu/
+Suites: noble noble-updates noble-backports
+Components: main universe restricted multiverse
+Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
+
+## Ubuntu security updates. Aside from URIs and Suites,
+## this should mirror your choices in the previous section.
+Types: deb-src
+URIs: http://security.ubuntu.com/ubuntu/
+Suites: noble-security
+Components: main universe restricted multiverse
+Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
+

helper.linux.fish

@@ -22,7 +22,7 @@ else
 end
 
 set -gx UBUNTUBUILDIMAGE_312_NAME arangodb/ubuntubuildarangodb-devel
-set -gx UBUNTUBUILDIMAGE_312_TAG 7
+set -gx UBUNTUBUILDIMAGE_312_TAG 8
 set -gx UBUNTUBUILDIMAGE_312 $UBUNTUBUILDIMAGE_312_NAME:$UBUNTUBUILDIMAGE_312_TAG-$UBUNTUBUILDIMAGE_TAG_ARCH
 
 set -gx UBUNTUBUILDIMAGE_311_NAME $UBUNTUBUILDIMAGE_312_NAME

scripts/buildArangoDB312.fish

@@ -37,6 +37,12 @@ if test "$ARCH" = "x86_64" -a (string sub -s 1 -l 1 "$OPENSSLPATH") = "3"
   set -xg X86_64_SUFFIX "64"
 end
 
+if test "$ARCH" = "x86_64"
+  set -xg LAPACK_LIB_PATH "/usr/lib/x86_64-linux-gnu/lapack/liblapack.a"
+else
+  set -xg LAPACK_LIB_PATH "/usr/lib/aarch64-linux-gnu/lapack/liblapack.a"
+end
+
 set -l pie ""
 
 if test "$STATIC_EXECUTABLES" = ""
@@ -53,15 +59,16 @@ set -g FULLARGS $argv \
  -DOPENSSL_ROOT_DIR=/opt \
  -DUSE_STRICT_OPENSSL_VERSION=$USE_STRICT_OPENSSL \
  -DBUILD_REPO_INFO=$BUILD_REPO_INFO \
- -DARANGODB_BUILD_DATE="$ARANGODB_BUILD_DATE"
+ -DARANGODB_BUILD_DATE="$ARANGODB_BUILD_DATE" \
+ -DLAPACK_LIBRARIES="$LAPACK_LIB_PATH"
 
 if test "$MAINTAINER" = "On"
   set -g FULLARGS $FULLARGS \
-    -DCMAKE_EXE_LINKER_FLAGS="-Wl,--build-id=sha1 $pie -fno-stack-protector -fuse-ld=lld" \
+    -DCMAKE_EXE_LINKER_FLAGS="-Wl,--build-id=sha1 $pie -fno-stack-protector -fuse-ld=lld -fopenmp=libomp -L/opt/omp -llapack -lgfortran" \
     -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld"
 else
   set -g FULLARGS $FULLARGS \
-    -DCMAKE_EXE_LINKER_FLAGS="-Wl,--build-id=sha1 $pie $inline -fno-stack-protector -fuse-ld=lld " \
+    -DCMAKE_EXE_LINKER_FLAGS="-Wl,--build-id=sha1 $pie $inline -fno-stack-protector -fuse-ld=lld -fopenmp=libomp -L/opt/omp -llapack -lgfortran" \
     -DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld" \
     -DUSE_CATCH_TESTS=Off \
     -DUSE_GOOGLE_TESTS=Off