F.parallel stl sort

CMakeLists.txt

@@ -110,7 +110,14 @@ if (USE_PARALLEL)
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
         set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++")
         add_definitions("-D_PARALLEL_SORT")
+
     endif ()
+
+    find_package(TBB REQUIRED)
+    message("TBB libraries found: ${TBB_LIBRARIES}")
+    link_libraries(tbb)
+
+
 endif()
 
 if (USE_TREE_BASED_CACHE)

Dockerfile

@@ -7,6 +7,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 FROM base as builder
 RUN apt-get update && apt-get install -y build-essential cmake libsparsehash-dev libicu-dev tzdata
+RUN apt install -y libtbb-dev
 COPY . /app/
 
 WORKDIR /app/
@@ -18,6 +19,7 @@ RUN cmake -DCMAKE_BUILD_TYPE=Release -DLOGLEVEL=DEBUG -DUSE_PARALLEL=true .. &&
 FROM base as runtime
 WORKDIR /app
 RUN apt-get update && apt-get install -y wget python3-yaml unzip curl bzip2 pkg-config libicu-dev python3-icu libgomp1
+RUN apt install -y libtbb-dev
 
 ARG UID=1000
 RUN groupadd -r qlever && useradd --no-log-init -r -u $UID -g qlever qlever && chown qlever:qlever /app

src/engine/CMakeLists.txt

@@ -37,3 +37,7 @@ add_library(engine
         )
 
 target_link_libraries(engine index parser SortPerformanceEstimator absl::flat_hash_set)
+
+if (USE_PARALLEL)
+  target_link_libraries(engine tbb)
+endif()

src/engine/Engine.h

@@ -101,12 +101,18 @@ class Engine {
     LOG(DEBUG) << "Sorting " << tab->size() << " elements ..." << std::endl;
     IdTableStatic<WIDTH> stab = tab->moveToStatic<WIDTH>();
     if constexpr (USE_PARALLEL_SORT) {
-      ad_utility::parallel_sort(
-          stab.begin(), stab.end(),
-          [keyColumn](const auto& a, const auto& b) {
-            return a[keyColumn] < b[keyColumn];
-          },
-          ad_utility::parallel_tag(NUM_SORT_THREADS));
+      auto comp = [keyColumn](const auto& a, const auto& b) {
+        return a[keyColumn] < b[keyColumn];
+      };
+      if constexpr (WIDTH != 0) {
+        // For the static Id tables, use the parallel Sort from the STL
+        ad_utility::parallel_stl_sort(stab.begin(), stab.end(), comp);
+      } else {
+        // For the dynamic IdTable, parallel Sorting is done via the gcc
+        // extension, since the stl sort requires default constructible rows
+        ad_utility::parallel_sort(stab.begin(), stab.end(), comp,
+                                  ad_utility::parallel_tag(NUM_SORT_THREADS));
+      }
     } else {
       std::sort(stab.begin(), stab.end(),
                 [keyColumn](const auto& a, const auto& b) {
@@ -129,8 +135,15 @@ class Engine {
     LOG(DEBUG) << "Sorting " << tab->size() << " elements.\n";
     IdTableStatic<WIDTH> stab = tab->moveToStatic<WIDTH>();
     if constexpr (USE_PARALLEL_SORT) {
-      ad_utility::parallel_sort(stab.begin(), stab.end(), comp,
-                                ad_utility::parallel_tag(NUM_SORT_THREADS));
+      if constexpr (WIDTH != 0) {
+        // For the static Id tables, use the parallel Sort from the STL
+        ad_utility::parallel_stl_sort(stab.begin(), stab.end(), comp);
+      } else {
+        // For the dynamic IdTable, parallel Sorting is done via the gcc
+        // extension, since the stl sort requires default constructible rows
+        ad_utility::parallel_sort(stab.begin(), stab.end(), comp,
+                                  ad_utility::parallel_tag(NUM_SORT_THREADS));
+      }
     } else {
       std::sort(stab.begin(), stab.end(), comp);
     }

src/global/Constants.h

@@ -5,6 +5,7 @@
 
 #include <stdexcept>
 #include <string>
+#include <algorithm>
 
 static const size_t STXXL_MEMORY_TO_USE = 1024L * 1024L * 1024L * 2L;
 static const size_t STXXL_DISK_SIZE_INDEX_BUILDER = 1000 * 1000;
@@ -121,6 +122,7 @@ static constexpr size_t PERCENTAGE_OF_TRIPLES_FOR_SORT_ESTIMATE = 5;
 
 #ifdef _PARALLEL_SORT
 static constexpr bool USE_PARALLEL_SORT = true;
+#include <execution>
 #include <parallel/algorithm>
 namespace ad_utility {
 template <typename... Args>
@@ -129,6 +131,11 @@ auto parallel_sort(Args&&... args) {
 }
 using parallel_tag = __gnu_parallel::parallel_tag;
 
+template <typename... Args>
+void parallel_stl_sort(Args&&... args) {
+  std::sort(std::execution::par_unseq, std::forward<Args>(args)...);
+}
+
 }  // namespace ad_utility
 
 #else
@@ -140,6 +147,12 @@ auto parallel_sort([[maybe_unused]] Args&&... args) {
       "Triggered the parallel sort although it was disabled. Please report to "
       "the developers!");
 }
+
+template <typename... Args>
+auto parallel_stl_sort([[maybe_unused]] Args&&... args) {
+  std::sort(std::forward<Args>(args)...);
+}
+
 using parallel_tag = int;
 }  // namespace ad_utility
 #endif

src/index/VocabularyGeneratorImpl.h

@@ -338,19 +338,9 @@ ItemVec vocabMapsToVector(std::shared_ptr<const ItemMapArray> map) {
 // _______________________________________________________________________________________________________________________
 template <class StringSortComparator>
 void sortVocabVector(ItemVec* vecPtr, StringSortComparator comp,
-                     const bool doParallelSort) {
+                     [[maybe_unused]] const bool doParallelSort) {
   auto& els = *vecPtr;
-  if constexpr (USE_PARALLEL_SORT) {
-    if (doParallelSort) {
-      ad_utility::parallel_sort(begin(els), end(els), comp,
-                                ad_utility::parallel_tag(NUM_SORT_THREADS));
-    } else {
-      std::sort(begin(els), end(els), comp);
-    }
-  } else {
-    std::sort(begin(els), end(els), comp);
-    (void)doParallelSort;  // avoid compiler warning for unused value.
-  }
+  ad_utility::parallel_stl_sort(begin(els), end(els), comp);
 }
 
 // _____________________________________________________________________