Merge pull request #873 from cwida/tpchext

TPC-H Extension
duckdb · Sep 2, 2020 · 2d8c5d9 · 2d8c5d9
2 parents d9bcedd + 3e9f1c2
commit 2d8c5d9
Show file tree

Hide file tree

Showing 354 changed files with 17,900 additions and 17,505 deletions.
diff --git a/.gitignore b/.gitignore
@@ -62,7 +62,6 @@ aclocal.m4
 config.log
 config.status
 stamp-h1
-config.h
 m4/libtool.m4
 m4/ltoptions.m4
 m4/ltsugar.m4

diff --git a/.travis.yml b/.travis.yml
@@ -36,7 +36,7 @@ matrix:
       script:
         - mkdir -p build/release
         - python3 scripts/amalgamation.py
-        - (cd build/release && cmake -DCMAKE_BUILD_TYPE=Release -DDISABLE_UNITY=1 -DBUILD_ICU_EXTENSION=1 -DBUILD_PARQUET_EXTENSION=1  -DJDBC_DRIVER=1 ../.. && cmake --build .)
+        - (cd build/release && cmake -DCMAKE_BUILD_TYPE=Release -DDISABLE_UNITY=1 -DBUILD_ICU_EXTENSION=1 -DBUILD_PARQUET_EXTENSION=1 -DBUILD_TPCH_EXTENSION=1 -DJDBC_DRIVER=1 ../.. && cmake --build .)
 
         - build/release/test/unittest "*"
         - python3.7 tools/shell/shell-test.py build/release/duckdb
@@ -61,7 +61,7 @@ matrix:
       script:
         - mkdir -p build/release
         - python3 scripts/amalgamation.py
-        - (cd build/release && cmake -DCMAKE_BUILD_TYPE=Release -DDISABLE_UNITY=1 -DBUILD_ICU_EXTENSION=1 -DBUILD_PARQUET_EXTENSION=1 -DJDBC_DRIVER=1 ../.. && travis_wait 30 cmake --build .)
+        - (cd build/release && cmake -DCMAKE_BUILD_TYPE=Release -DDISABLE_UNITY=1 -DBUILD_ICU_EXTENSION=1 -DBUILD_PARQUET_EXTENSION=1 -DBUILD_TPCH_EXTENSION=1 -DJDBC_DRIVER=1 ../.. && travis_wait 30 cmake --build .)
 
         - build/release/test/unittest
         - python3 tools/shell/shell-test.py build/release/duckdb
@@ -86,7 +86,7 @@ matrix:
 
       script:
         - C:/Python37-x64/python.exe scripts/amalgamation.py
-        - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_GENERATOR_PLATFORM=x64 -DJDBC_DRIVER=1 -DBUILD_ICU_EXTENSION=1 -DBUILD_PARQUET_EXTENSION=1 -DDISABLE_UNITY=1
+        - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_GENERATOR_PLATFORM=x64 -DJDBC_DRIVER=1 -DBUILD_ICU_EXTENSION=1 -DBUILD_PARQUET_EXTENSION=1 -DBUILD_TPCH_EXTENSION=1 -DDISABLE_UNITY=1
         - cmake --build . --target duckdb --config Release
         - cmake --build . --target unittest --config Release
         - cmake --build . --target shell --config Release

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -29,8 +29,12 @@ if(CCACHE_PROGRAM)
 endif()
 
 # Determine install paths
-set(INSTALL_LIB_DIR lib CACHE PATH "Installation directory for libraries")
-set(INSTALL_BIN_DIR bin CACHE PATH "Installation directory for executables")
+set(INSTALL_LIB_DIR
+    lib
+    CACHE PATH "Installation directory for libraries")
+set(INSTALL_BIN_DIR
+    bin
+    CACHE PATH "Installation directory for executables")
 set(INSTALL_INCLUDE_DIR
     include
     CACHE PATH "Installation directory for header files")
@@ -45,11 +49,7 @@ set(INSTALL_CMAKE_DIR
 set(DUCKDB_EXPORT_SET "DuckDBExports")
 
 # Make relative install paths absolute
-foreach(p
-        LIB
-        BIN
-        INCLUDE
-        CMAKE)
+foreach(p LIB BIN INCLUDE CMAKE)
   set(var INSTALL_${p}_DIR)
   if(NOT IS_ABSOLUTE "${${var}}")
     set(${var} "${CMAKE_INSTALL_PREFIX}/${${var}}")
@@ -75,9 +75,12 @@ endif()
 
 option(ENABLE_SANITIZER "Enable sanitizer." TRUE)
 if(${ENABLE_SANITIZER})
-  if (BUILD_PYTHON OR BUILD_R)
-    if ("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
-      message(WARNING "Sanitizers are not supported for the Python build. This is because sanitizers need to be compiled into the entire program, they cannot be added using dlopen, hence they do not work inside Python packages unless Python itself is compiled with sanitizer flags enabled")
+  if(BUILD_PYTHON OR BUILD_R)
+    if("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
+      message(
+        WARNING
+          "Sanitizers are not supported for the Python build. This is because sanitizers need to be compiled into the entire program, they cannot be added using dlopen, hence they do not work inside Python packages unless Python itself is compiled with sanitizer flags enabled"
+      )
     endif()
   else()
     set(CXX_EXTRA_DEBUG "${CXX_EXTRA_DEBUG} -fsanitize=address")
@@ -96,20 +99,19 @@ if(${CMAKE_SYSTEM_NAME} STREQUAL "SunOS")
   set(SUN TRUE)
 endif()
 
-execute_process(COMMAND git
-                        log
-                        -1
-                        --format=%h
-                WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-                OUTPUT_VARIABLE GIT_COMMIT_HASH
-                OUTPUT_STRIP_TRAILING_WHITESPACE)
+execute_process(
+  COMMAND git log -1 --format=%h
+  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+  OUTPUT_VARIABLE GIT_COMMIT_HASH
+  OUTPUT_STRIP_TRAILING_WHITESPACE)
 
 option(AMALGAMATION_BUILD
        "Build from the amalgamation files, rather than from the normal sources."
        FALSE)
 
 option(BUILD_ICU_EXTENSION "Build the ICU extension." FALSE)
 option(BUILD_PARQUET_EXTENSION "Build the Parquet extension." FALSE)
+option(BUILD_TPCH_EXTENSION "Build the TPC-H extension." FALSE)
 option(BUILD_BENCHMARKS "Enable building of the benchmark suite." FALSE)
 option(BUILD_SQLSMITH "Enable building of SQLSmith." FALSE)
 option(BUILD_TPCE "Enable building of the TPC-E tool." FALSE)
@@ -118,9 +120,13 @@ option(BUILD_PYTHON "Build the DuckDB Python extension" FALSE)
 
 option(TREAT_WARNINGS_AS_ERRORS "Treat warnings as errors" FALSE)
 
-if (BUILD_PYTHON OR BUILD_R)
-set(BUILD_ICU_EXTENSION TRUE)
-set(BUILD_PARQUET_EXTENSION TRUE)
+if(BUILD_PYTHON OR BUILD_R)
+  set(BUILD_ICU_EXTENSION TRUE)
+  set(BUILD_PARQUET_EXTENSION TRUE)
+endif()
+
+if(BUILD_SQLSMITH)
+  set(BUILD_TPCH_EXTENSION TRUE)
 endif()
 
 if(TREAT_WARNINGS_AS_ERRORS)
@@ -134,16 +140,15 @@ if(NOT MSVC)
       "${CMAKE_CXX_FLAGS_RELEASE} -O3 -DNDEBUG ${M32_FLAG} ${CXX_EXTRA}")
   set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELEASE} -g")
 
-  set(
-    CXX_EXTRA_DEBUG
-    "${CXX_EXTRA_DEBUG} -Wunused-variable -Wunused-const-variable -Werror=vla -Wnarrowing"
-    )
+  set(CXX_EXTRA_DEBUG
+      "${CXX_EXTRA_DEBUG} -Wunused-variable -Wunused-const-variable -Werror=vla -Wnarrowing"
+  )
   if(TREAT_WARNINGS_AS_ERRORS)
     set(CXX_EXTRA_DEBUG "${CXX_EXTRA_DEBUG} -Werror")
   endif()
 
-  if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU"
-     AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 8.0)
+  if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION
+                                                   VERSION_GREATER 8.0)
     set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${CXX_EXTRA_DEBUG}")
   elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang"
          AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 9.0)
@@ -158,16 +163,11 @@ else()
     set(CMAKE_CXX_WINDOWS_FLAGS "${CMAKE_CXX_WINDOWS_FLAGS} /WX")
   endif()
   # remove warning from CXX flags
-  string(REGEX
-         REPLACE "/W[0-4]"
-                 ""
-                 CMAKE_CXX_FLAGS
-                 "${CMAKE_CXX_FLAGS}")
+  string(REGEX REPLACE "/W[0-4]" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
   # add to-be-ignored warnings
-  set(
-    CMAKE_CXX_FLAGS
-    "${CMAKE_CXX_FLAGS} /wd4244 /wd4267 /wd4200 /wd26451 /wd26495 /D_CRT_SECURE_NO_WARNINGS"
-    )
+  set(CMAKE_CXX_FLAGS
+      "${CMAKE_CXX_FLAGS} /wd4244 /wd4267 /wd4200 /wd26451 /wd26495 /D_CRT_SECURE_NO_WARNINGS"
+  )
 endif()
 
 # todo use CHECK_CXX_COMPILER_FLAG(-fsanitize=address SUPPORTS_SANITIZER) etc.
@@ -201,10 +201,7 @@ function(enable_unity_build UB_SUFFIX SOURCE_VARIABLE_NAME)
   set(unit_build_file ${CMAKE_CURRENT_BINARY_DIR}/ub_${UB_SUFFIX}.cpp)
   set(temp_unit_build_file ${CMAKE_CURRENT_BINARY_DIR}/ub_${UB_SUFFIX}.cpp.tmp)
   # Exclude all translation units from compilation
-  set_source_files_properties(${files}
-                              PROPERTIES
-                              HEADER_FILE_ONLY
-                              true)
+  set_source_files_properties(${files} PROPERTIES HEADER_FILE_ONLY true)
 
   set(rebuild FALSE)
   # check if any of the source files have changed
@@ -220,17 +217,15 @@ function(enable_unity_build UB_SUFFIX SOURCE_VARIABLE_NAME)
     file(
       APPEND ${temp_unit_build_file}
       "#line 0 \"${source_file}\"\n#include <${CMAKE_CURRENT_SOURCE_DIR}/${source_file}>\n"
-      )
+    )
   endforeach(source_file)
 
-  execute_process(COMMAND ${CMAKE_COMMAND}
-                          -E
-                          compare_files
-                          ${unit_build_file}
-                          ${temp_unit_build_file}
-                  RESULT_VARIABLE compare_result
-                  OUTPUT_VARIABLE bla
-                  ERROR_VARIABLE bla)
+  execute_process(
+    COMMAND ${CMAKE_COMMAND} -E compare_files ${unit_build_file}
+            ${temp_unit_build_file}
+    RESULT_VARIABLE compare_result
+    OUTPUT_VARIABLE bla
+    ERROR_VARIABLE bla)
   if(compare_result EQUAL 0)
     # files are identical: do nothing
   elseif(compare_result EQUAL 1)
@@ -247,7 +242,7 @@ function(enable_unity_build UB_SUFFIX SOURCE_VARIABLE_NAME)
       file(
         APPEND ${unit_build_file}
         "#line 0 \"${source_file}\"\n#include <${CMAKE_CURRENT_SOURCE_DIR}/${source_file}>\n"
-        )
+      )
     endforeach(source_file)
   endif()
 
@@ -274,13 +269,49 @@ function(disable_target_warnings NAME)
   endif()
 endfunction()
 
+function(add_extension_definitions)
+  include_directories(${CMAKE_SOURCE_DIR}/extension)
+
+  if(${BUILD_ICU_EXTENSION})
+    include_directories(${CMAKE_SOURCE_DIR}/extension/icu/include)
+    add_definitions(-DBUILD_ICU_EXTENSION=${BUILD_ICU_EXTENSION})
+  endif()
+
+  if(${BUILD_PARQUET_EXTENSION})
+    include_directories(${CMAKE_SOURCE_DIR}/extension/parquet/include)
+    add_definitions(-DBUILD_PARQUET_EXTENSION=${BUILD_PARQUET_EXTENSION})
+  endif()
+
+  if(${BUILD_TPCH_EXTENSION})
+    include_directories(${CMAKE_SOURCE_DIR}/extension/tpch/include)
+    add_definitions(-DBUILD_TPCH_EXTENSION=${BUILD_TPCH_EXTENSION})
+  endif()
+endfunction()
+
+function(link_extension_libraries LIBRARY)
+  if(${BUILD_PARQUET_EXTENSION})
+    target_link_libraries(${LIBRARY} parquet_extension)
+  endif()
+
+  if(${BUILD_ICU_EXTENSION})
+    target_link_libraries(${LIBRARY} icu_extension)
+  endif()
+
+  if(${BUILD_TPCH_EXTENSION})
+    target_link_libraries(${LIBRARY} tpch_extension)
+  endif()
+
+endfunction()
+
 add_subdirectory(extension)
 add_subdirectory(src)
 
 option(BUILD_UNITTESTS "Build the C++ Unit Tests." TRUE)
 if(${BUILD_UNITTESTS})
   add_subdirectory(test)
-  if(NOT WIN32 AND NOT SUN AND ${BUILD_BENCHMARKS})
+  if(NOT WIN32
+     AND NOT SUN
+     AND ${BUILD_BENCHMARKS})
     add_subdirectory(benchmark)
   endif()
 endif()
@@ -305,9 +336,7 @@ if(EXISTS ${CMAKE_CONFIG_TEMPLATE} AND EXISTS ${CMAKE_CONFIG_VERSION_TEMPLATE})
                  "${PROJECT_BINARY_DIR}/DuckDBConfig.cmake" @ONLY)
 
   # Configure cmake package config for the install tree
-  file(RELATIVE_PATH
-       REL_INCLUDE_DIR
-       "${INSTALL_CMAKE_DIR}"
+  file(RELATIVE_PATH REL_INCLUDE_DIR "${INSTALL_CMAKE_DIR}"
        "${INSTALL_INCLUDE_DIR}")
   set(CONF_INCLUDE_DIRS "\${DuckDB_CMAKE_DIR}/${REL_INCLUDE_DIR}")
   configure_file(

diff --git a/Makefile b/Makefile
@@ -29,6 +29,9 @@ endif
 ifeq (${BUILD_ICU}, 1)
 	EXTENSIONS:=${EXTENSIONS} -DBUILD_ICU_EXTENSION=1
 endif
+ifeq (${BUILD_TPCH}, 1)
+	EXTENSIONS:=${EXTENSIONS} -DBUILD_TPCH_EXTENSION=1
+endif
 ifeq (${BUILD_SQLSMITH}, 1)
 	EXTENSIONS:=${EXTENSIONS} -DBUILD_SQLSMITH=1
 endif

diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt
@@ -5,25 +5,21 @@ include_directories(../test/include)
 include_directories(include)
 
 add_subdirectory(micro)
-add_subdirectory(tpch)
+if(${BUILD_TPCH_EXTENSION})
+  add_subdirectory(tpch)
+endif()
 add_subdirectory(tpcds)
 add_subdirectory(imdb)
-add_subdirectory(expression_reordering)
+
+add_extension_definitions()
 
 add_executable(benchmark_runner benchmark_runner.cpp interpreted_benchmark.cpp
                                 ${BENCHMARK_OBJECT_FILES})
 
-target_link_libraries(benchmark_runner
-                      duckdb
-                      dbgen
-                      dsdgen
-                      imdb
-                      test_helpers)
+target_link_libraries(benchmark_runner duckdb dsdgen imdb test_helpers)
+
+link_extension_libraries(benchmark_runner)
 
 if(${BUILD_TPCE})
   target_link_libraries(benchmark_runner tpce)
 endif()
-
-if(${BUILD_PARQUET_EXTENSION})
-  target_link_libraries(benchmark_runner parquet_extension)
-endif()
diff --git a/benchmark/benchmark_runner.cpp b/benchmark/benchmark_runner.cpp
@@ -273,6 +273,8 @@ ConfigurationError run_benchmarks(const BenchmarkConfiguration &configuration) {
 		for (idx_t index = 0; index < benchmarks.size(); ++index) {
 			if (RE2::FullMatch(benchmarks[index]->name, configuration.name_pattern)) {
 				benchmark_indices.emplace_back(index);
+			} else if (RE2::FullMatch(benchmarks[index]->group, configuration.name_pattern)) {
+				benchmark_indices.emplace_back(index);
 			}
 		}
 		benchmark_indices.shrink_to_fit();

diff --git a/benchmark/expression_reordering/CMakeLists.txt b/benchmark/expression_reordering/CMakeLists.txt