[Buffer Manager] Create volatile region for virtual memory management

.gitignore

@@ -1,5 +1,4 @@
 obj/*
-imdb_data
 .DS_STORE
 build/*
 build-*/*
@@ -44,3 +43,6 @@ default.profraw
 tpcc_cached_tables/
 tpcds_cached_tables/
 tpch_cached_tables/
+imdb_data/
+jcch_data/
+ssb_data/

.gitmodules

@@ -53,9 +53,15 @@
 [submodule "third_party/sql-parser"]
 	path = third_party/sql-parser
 	url = https://github.com/hyrise/sql-parser.git
+[submodule "third_party/ssb-dbgen"]
+	path = third_party/ssb-dbgen
+	url = https://github.com/hyrise/ssb-dbgen.git
 [submodule "third_party/tpcds-kit"]
 	path = third_party/tpcds-kit
 	url = https://github.com/hyrise-mp/tpcds-kit.git
 [submodule "third_party/zstd"]
 	path = third_party/zstd
 	url = https://github.com/facebook/zstd.git
+[submodule "third_party/concurrentqueue"]
+	path = third_party/concurrentqueue
+	url = https://github.com/cameron314/concurrentqueue.git

CMakeLists.txt

@@ -1,5 +1,5 @@
 # consider updating DEPENDENCIES.md when you touch this line
-cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.18 FATAL_ERROR)
 
 project(Hyrise)
 
@@ -31,6 +31,12 @@ elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
         compiler_not_supported("Your clang version ${CMAKE_CXX_COMPILER_VERSION} is too old.")
     endif()
 elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang")
+    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14.0)
+        compiler_not_supported("Hyrise does not support Apple clang compilers older than version 14.")
+    endif()
+    # In release mode, we found issues on ARM with loop unrolling (macOS Ventura, Apple Clang 14.0).
+    # As we do not benchmark on macOS, we can safely ignore this issue for now.
+    add_compile_options(-Wno-pass-failed)
     execute_process(
         COMMAND brew --prefix
         RESULT_VARIABLE BREW
@@ -44,7 +50,7 @@ elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang")
     if (NOT EXISTS "${BREW_PREFIX}/opt/llvm/bin/clang")
         set(CMAKE_OSX_INSTALL_TEXT "install ${CMAKE_OSX_BREW_TEXT}the official llvm/clang package using `install_dependencies.sh` or `brew install llvm --with-toolchain` and then ")
     endif()
-    compiler_not_supported("Apple's clang compiler is not supported because it is lacking support for recent C++ features. Please ${CMAKE_OSX_INSTALL_TEXT}run cmake with `-DCMAKE_C_COMPILER=${BREW_PREFIX}/opt/llvm/bin/clang -DCMAKE_CXX_COMPILER=${BREW_PREFIX}/opt/llvm/bin/clang++`.")
+    message(WARNING "On macOS, we recommend a recent clang compiler instead of Apple's clang compiler. Please ${CMAKE_OSX_INSTALL_TEXT}run cmake with `-DCMAKE_C_COMPILER=${BREW_PREFIX}/opt/llvm/bin/clang -DCMAKE_CXX_COMPILER=${BREW_PREFIX}/opt/llvm/bin/clang++`.")
 else()
     compiler_not_supported("You are using an unsupported compiler (${CMAKE_CXX_COMPILER_ID})! Compilation has only been tested with Clang (Linux + OS X) and GCC (Linux).")
 endif()
@@ -105,17 +111,25 @@ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${FLAGS_ALL} ${FLAGS_REL
 set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} ${FLAGS_ALL} ${FLAGS_RELEASE}")
 set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${FLAGS_ALL} ${FLAGS_RELEASE}")
 
-# Check for unity builds
-if(${CMAKE_VERSION} VERSION_LESS "3.16.0")
-    if(DEFINED CMAKE_UNITY_BUILD)
-        message(FATAL_ERROR "-DCMAKE_UNITY_BUILD=ON was set, but your cmake version does not support unity builds.")
+include(CheckLinkerFlag)
+if (HYRISE_LINKER)
+    check_linker_flag(CXX "-fuse-ld=${HYRISE_LINKER}" HAS_HYRISE_LINKER)
+    if (NOT HAS_HYRISE_LINKER)
+        message(FATAL_ERROR "Got HYRISE_LINKER=${HYRISE_LINKER}, but -fuse-ld=${HYRISE_LINKER} does not work.")
+    endif()
+    message(STATUS "Using HYRISE_LINKER (${HYRISE_LINKER}).")
+    add_link_options("-fuse-ld=${HYRISE_LINKER}")
+else()
+    check_linker_flag(CXX "-fuse-ld=lld" HAS_LLD)
+    if (HAS_LLD)
+        message(STATUS "Found LLD, using as linker.")
+        add_link_options("-fuse-ld=lld")
+    else()
+        message(STATUS "Using default linker.")
+        if (NOT APPLE)
+            message(WARNING "Consider installing the LLD linker for faster incremental builds on Linux.")
+        endif()
     endif()
-endif()
-
-# On Linux, use the lld linker, which is twice as fast for incremental debug builds
-if (NOT APPLE)
-    set(CMAKE_SHARED_LINKER_FLAGS "-fuse-ld=lld")
-    set(CMAKE_EXE_LINKER_FLAGS "-fuse-ld=lld")
 endif()
 
 # Require NCurses over Curses
@@ -151,17 +165,14 @@ cmake_policy(SET CMP0069 NEW)
 set(CMAKE_POLICY_DEFAULT_CMP0069 NEW)
 option(ENABLE_LTO "Set to ON to build Hyrise with enabled Link Time Optimization (LTO). Default: OFF" OFF)
 if (${ENABLE_LTO})
-    if(NOT ${CMAKE_VERSION} VERSION_LESS "3.9.0")
-        cmake_minimum_required(VERSION 3.9 FATAL_ERROR)
-        include(CheckIPOSupported)
-        check_ipo_supported(RESULT ipo_supported OUTPUT ipo_output)
-        if(ipo_supported)
-            set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
-            add_definitions(-DWITH_LTO)
-            message(STATUS "Building with Link-Time Optimization")
-        else()
-            message(WARNING "LTO not supported: ${ipo_output}")
-        endif()
+    include(CheckIPOSupported)
+    check_ipo_supported(RESULT ipo_supported OUTPUT ipo_output)
+    if(ipo_supported)
+        set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
+        add_definitions(-DWITH_LTO)
+        message(STATUS "Building with Link-Time Optimization")
+    else()
+        message(WARNING "LTO not supported: ${ipo_output}")
     endif()
 endif()
 

DEPENDENCIES.md

@@ -8,15 +8,15 @@
 | clang-format              | >= 11.0          |    All   |                      Yes (formatting) |
 | clang-tidy                | >= 11.0          |    All   |                         Yes (linting) |
 | coreutils                 | any              |    Mac   |                         Yes (scripts) |
-| cmake                     | >= 3.9           |    All   |                                    No |
+| cmake                     | >= 3.18          |    All   |                                    No |
 | dos2unix                  | any              |    All   |                         Yes (linting) |
 | gcc                       | >= 9.1           |    All   | Yes, if clang installed, not for OS X |
 | gcovr                     | >= 3.2           |    All   |                        Yes (coverage) |
 | graphviz                  | any              |    All   |             Yes (query visualization) |
 | libnuma-dev               | any              |    Linux |                            Yes (numa) |
 | libnuma1                  | any              |    Linux |                            Yes (numa) |
 | libpq-dev                 | >= 9             |    All   |                                    No |
-| lld                       | any              |    Linux |   No, but could be removed from cmake |
+| lld                       | any              |    Linux |                                   Yes |
 | parallel                  | any              |    All   |                                   Yes |
 | pexpect                   | >= 4             |    All   |                     Yes (tests in CI) |
 | postgresql-server-dev-all | >= 154           |    Linux |                                    No |

Jenkinsfile

@@ -52,7 +52,7 @@ try {
       // Print the hostname to let us know on which node the docker image was executed for reproducibility.
       sh "hostname"
     }
-  
+
     // The empty '' results in using the default registry: https://index.docker.io/v1/
     docker.withRegistry('', 'docker') {
       def hyriseCI = docker.image('hyrise/hyrise-ci:22.04');
@@ -81,7 +81,7 @@ try {
 
             // We don't use unity builds with GCC 9 as it triggers https://github.com/google/googletest/issues/3552
             unity = '-DCMAKE_UNITY_BUILD=ON'
- 
+
             // With Hyrise, we aim to support the most recent compiler versions and do not invest a lot of work to
             // support older versions. We test the oldest LLVM version shipped with Ubuntu 22.04 (i.e., LLVM 11) and
             // GCC 9 (oldest version supported by Hyrise). We execute at least debug runs for them.
@@ -173,13 +173,15 @@ try {
                 sh "./scripts/test/hyriseBenchmarkJoinOrder_test.py clang-debug"
                 sh "./scripts/test/hyriseBenchmarkFileBased_test.py clang-debug"
                 sh "cd clang-debug && ../scripts/test/hyriseBenchmarkTPCH_test.py ." // Own folder to isolate visualization
-                sh "cd clang-debug && ../scripts/test/hyriseBenchmarkJCCH_test.py ." // Own folder to isolate visualization
+                sh "cd clang-debug && ../scripts/test/hyriseBenchmarkJCCH_test.py ." // Own folder to isolate cached data
+                sh "cd clang-debug && ../scripts/test/hyriseBenchmarkStarSchema_test.py ." // Own folder to isolate cached data
                 sh "./scripts/test/hyriseConsole_test.py gcc-debug"
                 sh "./scripts/test/hyriseServer_test.py gcc-debug"
                 sh "./scripts/test/hyriseBenchmarkJoinOrder_test.py gcc-debug"
                 sh "./scripts/test/hyriseBenchmarkFileBased_test.py gcc-debug"
                 sh "cd gcc-debug && ../scripts/test/hyriseBenchmarkTPCH_test.py ." // Own folder to isolate visualization
-                sh "cd gcc-debug && ../scripts/test/hyriseBenchmarkJCCH_test.py ." // Own folder to isolate visualization
+                sh "cd gcc-debug && ../scripts/test/hyriseBenchmarkJCCH_test.py ." // Own folder to isolate cached data
+                sh "cd gcc-debug && ../scripts/test/hyriseBenchmarkStarSchema_test.py ." // Own folder to isolate cached data
 
               } else {
                 Utils.markStageSkippedForConditional("debugSystemTests")
@@ -255,7 +257,7 @@ try {
                 sh "cd clang-release-addr-ub-sanitizers && make hyriseTest hyriseSystemTest hyriseBenchmarkTPCH hyriseBenchmarkTPCC -j \$(( \$(nproc) / 10))"
                 sh "LSAN_OPTIONS=suppressions=resources/.lsan-ignore.txt ASAN_OPTIONS=suppressions=resources/.asan-ignore.txt ./clang-release-addr-ub-sanitizers/hyriseTest clang-release-addr-ub-sanitizers"
                 sh "LSAN_OPTIONS=suppressions=resources/.lsan-ignore.txt ASAN_OPTIONS=suppressions=resources/.asan-ignore.txt ./clang-release-addr-ub-sanitizers/hyriseSystemTest ${tests_excluded_in_sanitizer_builds} clang-release-addr-ub-sanitizers"
-                sh "LSAN_OPTIONS=suppressions=resources/.lsan-ignore.txt ASAN_OPTIONS=suppressions=resources/.asan-ignore.txt ./clang-release-addr-ub-sanitizers/hyriseBenchmarkTPCH -s .01 --verify -r 100 --scheduler --clients 10"
+                sh "LSAN_OPTIONS=suppressions=resources/.lsan-ignore.txt ASAN_OPTIONS=suppressions=resources/.asan-ignore.txt ./clang-release-addr-ub-sanitizers/hyriseBenchmarkTPCH -s .01 --verify -r 100 --scheduler --clients 10 --cores \$(( \$(nproc) / 4))"
                 sh "cd clang-release-addr-ub-sanitizers && LSAN_OPTIONS=suppressions=resources/.lsan-ignore.txt ASAN_OPTIONS=suppressions=resources/.asan-ignore.txt ../scripts/test/hyriseBenchmarkTPCC_test.py ." // Own folder to isolate binary export tests
               } else {
                 Utils.markStageSkippedForConditional("clangReleaseAddrUBSanitizers")
@@ -267,7 +269,7 @@ try {
                 sh "cd clang-relwithdebinfo-thread-sanitizer && make hyriseTest hyriseSystemTest hyriseBenchmarkTPCH -j \$(( \$(nproc) / 10))"
                 sh "TSAN_OPTIONS=\"history_size=7 suppressions=resources/.tsan-ignore.txt\" ./clang-relwithdebinfo-thread-sanitizer/hyriseTest clang-relwithdebinfo-thread-sanitizer"
                 sh "TSAN_OPTIONS=\"history_size=7 suppressions=resources/.tsan-ignore.txt\" ./clang-relwithdebinfo-thread-sanitizer/hyriseSystemTest ${tests_excluded_in_sanitizer_builds} clang-relwithdebinfo-thread-sanitizer"
-                sh "TSAN_OPTIONS=\"history_size=7 suppressions=resources/.tsan-ignore.txt\" ./clang-relwithdebinfo-thread-sanitizer/hyriseBenchmarkTPCH -s .01 --verify -r 100 --scheduler --clients 10"
+                sh "TSAN_OPTIONS=\"history_size=7 suppressions=resources/.tsan-ignore.txt\" ./clang-relwithdebinfo-thread-sanitizer/hyriseBenchmarkTPCH -s .01 --verify -r 100 --scheduler --clients 10 --cores \$(( \$(nproc) / 10))"
               } else {
                 Utils.markStageSkippedForConditional("clangRelWithDebInfoThreadSanitizer")
               }
@@ -351,7 +353,18 @@ try {
                 Utils.markStageSkippedForConditional("jobQueryPlans")
               }
             }
+          }, ssbQueryPlans: {
+            stage("ssbQueryPlans") {
+              if (env.BRANCH_NAME == 'master' || full_ci) {
+                sh "mkdir -p query_plans/ssb; cd query_plans/ssb && ../../clang-release/hyriseBenchmarkStarSchema --dont_cache_binary_tables -r 1 -s 1 --visualize && ../../scripts/plot_operator_breakdown.py ../../clang-release/"
+                archiveArtifacts artifacts: 'query_plans/ssb/*.svg'
+                archiveArtifacts artifacts: 'query_plans/ssb/operator_breakdown.pdf'
+              } else {
+                Utils.markStageSkippedForConditional("ssbQueryPlans")
+              }
+            }
           }
+
         } finally {
           sh "ls -A1 | xargs rm -rf"
           deleteDir()
@@ -363,25 +376,38 @@ try {
   parallel clangDebugMacX64: {
     node('mac') {
       stage("clangDebugMacX64") {
-        if (env.BRANCH_NAME == 'master' || full_ci) {
-          try {
-            checkout scm
+        // We have experienced frequent network problems with this CI machine. So far, we have not found the cause.
+        // Since we run this stage very late and it frequently fails due to network problems, we retry the stage three
+        // times as (i) we can be rather sure that most problems with the current pull request have already been found
+        // in earlier stages and fails in this stage are probably network issues, and (ii) we avoid re-runs of entire
+        // Full CI runs that failed in the very last stage due to a single network issue.
+        retry(3) {
+          if (env.BRANCH_NAME == 'master' || full_ci) {
+            try {
+              checkout scm
 
-            // We do not use install_dependencies.sh here as there is no way to run OS X in a Docker container
-            sh "git submodule update --init --recursive --jobs 4 --depth=1"
+              // We do not use install_dependencies.sh here as there is no way to run OS X in a Docker container
+              sh "git submodule update --init --recursive --jobs 4 --depth=1"
 
-            sh "mkdir clang-debug && cd clang-debug && /usr/local/bin/cmake ${debug} ${unity} -DCMAKE_C_COMPILER=/usr/local/opt/llvm@15/bin/clang -DCMAKE_CXX_COMPILER=/usr/local/opt/llvm@15/bin/clang++ .."
-            sh "cd clang-debug && make -j \$(sysctl -n hw.logicalcpu)"
-            sh "./clang-debug/hyriseTest"
-            sh "./clang-debug/hyriseSystemTest --gtest_filter=\"-TPCCTest*:TPCDSTableGeneratorTest.*:TPCHTableGeneratorTest.RowCountsMediumScaleFactor:*.CompareToSQLite/Line1*WithLZ4\""
-            sh "PATH=/usr/local/bin/:$PATH ./scripts/test/hyriseConsole_test.py clang-debug"
-            sh "PATH=/usr/local/bin/:$PATH ./scripts/test/hyriseServer_test.py clang-debug"
-            sh "PATH=/usr/local/bin/:$PATH ./scripts/test/hyriseBenchmarkFileBased_test.py clang-debug"
-          } finally {
-            sh "ls -A1 | xargs rm -rf"
+              // Build hyriseTest with macOS's default compiler (Apple clang) and run it.
+              sh "mkdir clang-apple-debug && cd clang-apple-debug && /usr/local/bin/cmake ${debug} ${unity} .."
+              sh "cd clang-apple-debug && make -j \$(sysctl -n hw.logicalcpu)"
+              sh "./clang-apple-debug/hyriseTest"
+
+              // Build Hyrise with a recent clang compiler version (as recommended for Hyrise on macOS) and run various tests.
+              sh "mkdir clang-debug && cd clang-debug && /usr/local/bin/cmake ${debug} ${unity} -DCMAKE_C_COMPILER=/usr/local/opt/llvm@16/bin/clang -DCMAKE_CXX_COMPILER=/usr/local/opt/llvm@16/bin/clang++ .."
+              sh "cd clang-debug && make -j \$(sysctl -n hw.logicalcpu)"
+              sh "./clang-debug/hyriseTest"
+              sh "./clang-debug/hyriseSystemTest --gtest_filter=\"-TPCCTest*:TPCDSTableGeneratorTest.*:TPCHTableGeneratorTest.RowCountsMediumScaleFactor:*.CompareToSQLite/Line1*WithLZ4\""
+              sh "PATH=/usr/local/bin/:$PATH ./scripts/test/hyriseConsole_test.py clang-debug"
+              sh "PATH=/usr/local/bin/:$PATH ./scripts/test/hyriseServer_test.py clang-debug"
+              sh "PATH=/usr/local/bin/:$PATH ./scripts/test/hyriseBenchmarkFileBased_test.py clang-debug"
+            } finally {
+              sh "ls -A1 | xargs rm -rf"
+            }
+          } else {
+            Utils.markStageSkippedForConditional("clangDebugMacX64")
           }
-        } else {
-          Utils.markStageSkippedForConditional("clangDebugMacX64")
         }
       }
     }
@@ -392,12 +418,18 @@ try {
         if (env.BRANCH_NAME == 'master' || full_ci) {
           try {
             checkout scm
-            
+
             // We do not use install_dependencies.sh here as there is no way to run OS X in a Docker container
             sh "git submodule update --init --recursive --jobs 4 --depth=1"
-
+
+            // Build hyriseTest with macOS's default compiler (Apple clang) and run it.
+            sh "mkdir clang-apple-release && cd clang-apple-release && cmake ${release} .."
+            sh "cd clang-apple-release && make -j \$(sysctl -n hw.logicalcpu)"
+            sh "./clang-apple-release/hyriseTest"
+
+            // Build Hyrise with a recent clang compiler version (as recommended for Hyrise on macOS) and run various tests.
             // NOTE: These paths differ from x64 - brew on ARM uses /opt (https://docs.brew.sh/Installation)
-            sh "mkdir clang-release && cd clang-release && cmake ${release} -DCMAKE_C_COMPILER=/opt/homebrew/opt/llvm@15/bin/clang -DCMAKE_CXX_COMPILER=/opt/homebrew/opt/llvm@15/bin/clang++ .."
+            sh "mkdir clang-release && cd clang-release && cmake ${release} -DCMAKE_C_COMPILER=/opt/homebrew/opt/llvm@16/bin/clang -DCMAKE_CXX_COMPILER=/opt/homebrew/opt/llvm@16/bin/clang++ .."
             sh "cd clang-release && make -j \$(sysctl -n hw.logicalcpu)"
 
             // Check whether arm64 binaries are built to ensure that we are not accidentally running rosetta that