Skip to content

Commit

Permalink
Merge pull request #747 from cwida/parallelism
Browse files Browse the repository at this point in the history
Basic Parallelism Support
  • Loading branch information
Mytherin committed Jul 10, 2020
2 parents 0c69daf + 9ae259d commit ebd9140
Show file tree
Hide file tree
Showing 129 changed files with 6,204 additions and 870 deletions.
7 changes: 3 additions & 4 deletions .travis.yml
Expand Up @@ -34,7 +34,7 @@ matrix:
- mkdir -p build/release
- python3 scripts/amalgamation.py
- (cd build/release && cmake -DCMAKE_BUILD_TYPE=Release -DAMALGAMATION_BUILD=1 -DBUILD_ICU_EXTENSION=1 -DBUILD_PARQUET_EXTENSION=1 -DJDBC_DRIVER=1 ../.. && cmake --build .)

- build/release/test/unittest "*"
- python3.7 tools/shell/shell-test.py build/release/duckdb
- build/release/extension/icu/test/icu_test
Expand All @@ -43,7 +43,6 @@ matrix:
- java -cp build/release/tools/jdbc/duckdb_jdbc.jar org.duckdb.test.TestDuckDBJDBC
- (cd examples/embedded-c; make)
- (cd examples/embedded-c++; make)
- (cd examples/programmatic-querying; make)
- (cd examples/jdbc; make; make maven)

- zip -j duckdb_cli-linux-amd64.zip build/release/duckdb
Expand All @@ -61,7 +60,7 @@ matrix:
- mkdir -p build/release
- python3 scripts/amalgamation.py
- (cd build/release && cmake -DCMAKE_BUILD_TYPE=Release -DAMALGAMATION_BUILD=1 -DBUILD_ICU_EXTENSION=1 -DBUILD_PARQUET_EXTENSION=1 -DJDBC_DRIVER=1 ../.. && cmake --build .)

- build/release/test/unittest
- python3 tools/shell/shell-test.py build/release/duckdb
- java -cp build/release/tools/jdbc/duckdb_jdbc.jar org.duckdb.test.TestDuckDBJDBC
Expand Down Expand Up @@ -244,7 +243,7 @@ matrix:
before_script:
- choco install python3 --version 3.7.3 -y --params "/InstallDir:C:\Python37-x64"
- choco install curl -y --force
- choco install jdk8 --version 8.0.211 --x86 -y --force
- choco install jdk8 --version 8.0.211 --x86 -y --force
- choco install zip -y --force

script:
Expand Down
10 changes: 4 additions & 6 deletions CMakeLists.txt
Expand Up @@ -170,6 +170,7 @@ include_directories(third_party/re2)
include_directories(third_party/miniz)
include_directories(third_party/utf8proc/include)
include_directories(third_party/miniparquet)
include_directories(third_party/concurrentqueue)

# todo only regenerate ub file if one of the input files changed hack alert
function(enable_unity_build UB_SUFFIX SOURCE_VARIABLE_NAME)
Expand Down Expand Up @@ -252,33 +253,30 @@ function(disable_target_warnings NAME)
endif()
endfunction()


option(BUILD_ICU_EXTENSION "Build the ICU extension." FALSE)
if(${BUILD_ICU_EXTENSION})
add_definitions( -DBUILD_ICU_EXTENSION=${BUILD_ICU_EXTENSION} )
add_definitions(-DBUILD_ICU_EXTENSION=${BUILD_ICU_EXTENSION})
endif()

option(BUILD_PARQUET_EXTENSION "Build the Parquet extension." FALSE)
if(${BUILD_PARQUET_EXTENSION})
add_definitions( -DBUILD_PARQUET_EXTENSION=${BUILD_PARQUET_EXTENSION} )
add_definitions(-DBUILD_PARQUET_EXTENSION=${BUILD_PARQUET_EXTENSION})
endif()


add_subdirectory(extension)
add_subdirectory(src)

option(BUILD_UNITTESTS "Build the C++ Unit Tests." TRUE)
if(${BUILD_UNITTESTS})
add_subdirectory(test)
if(NOT WIN32 AND NOT SUN)
add_subdirectory(benchmark)
add_subdirectory(benchmark)
endif()
endif()

add_subdirectory(third_party)
add_subdirectory(tools)


# Write the export set for build and install tree
install(EXPORT "${DUCKDB_EXPORT_SET}" DESTINATION "${INSTALL_CMAKE_DIR}")
export(EXPORT "${DUCKDB_EXPORT_SET}"
Expand Down
6 changes: 6 additions & 0 deletions Makefile
Expand Up @@ -56,6 +56,12 @@ release:
cmake $(GENERATOR) $(FORCE_COLOR) ${WARNINGS_AS_ERRORS} ${DISABLE_UNITY_FLAG} -DCMAKE_BUILD_TYPE=Release -DAMALGAMATION_BUILD=1 ../.. && \
cmake --build .

reldebug:
mkdir -p build/reldebug && \
cd build/reldebug && \
cmake $(GENERATOR) $(FORCE_COLOR) ${WARNINGS_AS_ERRORS} ${DISABLE_UNITY_FLAG} -DCMAKE_BUILD_TYPE=RelWithDebInfo ../.. && \
cmake --build .

amaldebug:
mkdir -p build/amaldebug && \
python scripts/amalgamation.py && \
Expand Down
13 changes: 8 additions & 5 deletions benchmark/tpch/CMakeLists.txt
Expand Up @@ -2,15 +2,18 @@ include_directories(../../third_party/dbgen/include)
include_directories(../../third_party/sqlite/include)
include_directories(../../extension/parquet/include)

set(BENCHMARK_SOURCE_FILES lineitem_aggregate.cpp sf1.cpp read_lineitem.cpp
startup.cpp)
set(BENCHMARK_SOURCE_FILES
lineitem_aggregate.cpp
sf1.cpp
sf1-parallel.cpp
read_lineitem.cpp
startup.cpp)

if(${BUILD_PARQUET_EXTENSION})
set(BENCHMARK_SOURCE_FILES ${BENCHMARK_SOURCE_FILES} lineitem_parquet.cpp)
endif()

add_library(duckdb_benchmark_tpch OBJECT ${BENCHMARK_SOURCE_FILES})

set(BENCHMARK_OBJECT_FILES
${BENCHMARK_OBJECT_FILES} $<TARGET_OBJECTS:duckdb_benchmark_tpch>
PARENT_SCOPE)
set(BENCHMARK_OBJECT_FILES ${BENCHMARK_OBJECT_FILES}
$<TARGET_OBJECTS:duckdb_benchmark_tpch> PARENT_SCOPE)
119 changes: 119 additions & 0 deletions benchmark/tpch/sf1-parallel.cpp
@@ -0,0 +1,119 @@
#include "benchmark_runner.hpp"
#include "compare_result.hpp"
#include "dbgen.hpp"
#include "duckdb_benchmark_macro.hpp"

using namespace duckdb;
using namespace std;

#define SF 1

#define TPCH_QUERY_BODY_PARALLEL(QNR) \
virtual void Load(DuckDBBenchmarkState *state) { \
Connection con(state->db); \
con.Query("PRAGMA threads=4"); \
string cached_name = "tpch_sf" + to_string(SF); \
if (!BenchmarkRunner::TryLoadDatabase(state->db, cached_name)) { \
tpch::dbgen(SF, state->db); \
BenchmarkRunner::SaveDatabase(state->db, cached_name); \
} \
} \
virtual string GetQuery() { \
return tpch::get_query(QNR); \
} \
virtual string VerifyResult(QueryResult *result) { \
if (!result->success) { \
return result->error; \
} \
return compare_csv(*result, tpch::get_answer(SF, QNR), true); \
} \
virtual string BenchmarkInfo() { \
return StringUtil::Format("TPC-H Q%d SF%d: %s", QNR, SF, tpch::get_query(QNR).c_str()); \
}

DUCKDB_BENCHMARK(Q01_PARALLEL, "[tpch-sf1-parallel]")
TPCH_QUERY_BODY_PARALLEL(1);
FINISH_BENCHMARK(Q01_PARALLEL)

DUCKDB_BENCHMARK(Q02_PARALLEL, "[tpch-sf1-parallel]")
TPCH_QUERY_BODY_PARALLEL(2);
FINISH_BENCHMARK(Q02_PARALLEL)

DUCKDB_BENCHMARK(Q03_PARALLEL, "[tpch-sf1-parallel]")
TPCH_QUERY_BODY_PARALLEL(3);
FINISH_BENCHMARK(Q03_PARALLEL)

DUCKDB_BENCHMARK(Q04_PARALLEL, "[tpch-sf1-parallel]")
TPCH_QUERY_BODY_PARALLEL(4);
FINISH_BENCHMARK(Q04_PARALLEL)

DUCKDB_BENCHMARK(Q05_PARALLEL, "[tpch-sf1-parallel]")
TPCH_QUERY_BODY_PARALLEL(5);
FINISH_BENCHMARK(Q05_PARALLEL)

DUCKDB_BENCHMARK(Q06_PARALLEL, "[tpch-sf1-parallel]")
TPCH_QUERY_BODY_PARALLEL(6);
FINISH_BENCHMARK(Q06_PARALLEL)

DUCKDB_BENCHMARK(Q07_PARALLEL, "[tpch-sf1-parallel]")
TPCH_QUERY_BODY_PARALLEL(7);
FINISH_BENCHMARK(Q07_PARALLEL)

DUCKDB_BENCHMARK(Q08_PARALLEL, "[tpch-sf1-parallel]")
TPCH_QUERY_BODY_PARALLEL(8);
FINISH_BENCHMARK(Q08_PARALLEL)

DUCKDB_BENCHMARK(Q09_PARALLEL, "[tpch-sf1-parallel]")
TPCH_QUERY_BODY_PARALLEL(9);
FINISH_BENCHMARK(Q09_PARALLEL)

DUCKDB_BENCHMARK(Q10_PARALLEL, "[tpch-sf1-parallel]")
TPCH_QUERY_BODY_PARALLEL(10);
FINISH_BENCHMARK(Q10_PARALLEL)

DUCKDB_BENCHMARK(Q11_PARALLEL, "[tpch-sf1-parallel]")
TPCH_QUERY_BODY_PARALLEL(11);
FINISH_BENCHMARK(Q11_PARALLEL)

DUCKDB_BENCHMARK(Q12_PARALLEL, "[tpch-sf1-parallel]")
TPCH_QUERY_BODY_PARALLEL(12);
FINISH_BENCHMARK(Q12_PARALLEL)

DUCKDB_BENCHMARK(Q13_PARALLEL, "[tpch-sf1-parallel]")
TPCH_QUERY_BODY_PARALLEL(13);
FINISH_BENCHMARK(Q13_PARALLEL)

DUCKDB_BENCHMARK(Q14_PARALLEL, "[tpch-sf1-parallel]")
TPCH_QUERY_BODY_PARALLEL(14);
FINISH_BENCHMARK(Q14_PARALLEL)

DUCKDB_BENCHMARK(Q15_PARALLEL, "[tpch-sf1-parallel]")
TPCH_QUERY_BODY_PARALLEL(15);
FINISH_BENCHMARK(Q15_PARALLEL)

DUCKDB_BENCHMARK(Q16_PARALLEL, "[tpch-sf1-parallel]")
TPCH_QUERY_BODY_PARALLEL(16);
FINISH_BENCHMARK(Q16_PARALLEL)

DUCKDB_BENCHMARK(Q17_PARALLEL, "[tpch-sf1-parallel]")
TPCH_QUERY_BODY_PARALLEL(17);
FINISH_BENCHMARK(Q17_PARALLEL)

DUCKDB_BENCHMARK(Q18_PARALLEL, "[tpch-sf1-parallel]")
TPCH_QUERY_BODY_PARALLEL(18);
FINISH_BENCHMARK(Q18_PARALLEL)

DUCKDB_BENCHMARK(Q19_PARALLEL, "[tpch-sf1-parallel]")
TPCH_QUERY_BODY_PARALLEL(19);
FINISH_BENCHMARK(Q19_PARALLEL)

DUCKDB_BENCHMARK(Q20_PARALLEL, "[tpch-sf1-parallel]")
TPCH_QUERY_BODY_PARALLEL(20);
FINISH_BENCHMARK(Q20_PARALLEL)

DUCKDB_BENCHMARK(Q21_PARALLEL, "[tpch-sf1-parallel]")
TPCH_QUERY_BODY_PARALLEL(21);
FINISH_BENCHMARK(Q21_PARALLEL)
DUCKDB_BENCHMARK(Q22_PARALLEL, "[tpch-sf1-parallel]")
TPCH_QUERY_BODY_PARALLEL(22);
FINISH_BENCHMARK(Q22_PARALLEL)
10 changes: 0 additions & 10 deletions examples/programmatic-querying/CMakeLists.txt

This file was deleted.

18 changes: 0 additions & 18 deletions examples/programmatic-querying/Makefile

This file was deleted.

0 comments on commit ebd9140

Please sign in to comment.