diff --git a/.gitignore b/.gitignore index 2a2f6e2..f4e2059 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ +venv .idea -*.json \ No newline at end of file +*.json +*.png diff --git a/README.md b/README.md index b772f3e..e405e6c 100644 --- a/README.md +++ b/README.md @@ -48,17 +48,12 @@ prohibited by the TPC. ## Data Generation -See the benchmark-specific instructions for generating the CSV data for [TPC-H](tpch) and [TPC-DS](tpcds). - -## Converting CSV data to Parquet - -Although it is valid to run benchmarks against CSV data, this does not really represent how most of the world is -running OLAP queries, especially when dealing with large datasets. When benchmarking DataFusion, we typically want -to be querying Parquet data, so first we must convert the generated datasets to Parquet. Also, we typically do not -want a single file per table, so we also need to repartition the data. - -We plan on adding Python scripts in this repository to perform this conversion and repartitioning. Until then you may -want to write your own scripts using DataFusion or Spark. Another option is to use [tpc-tools](https://crates.io/crates/tpctools). +See the benchmark-specific instructions for generating the CSV data for [TPC-H](tpch) and [TPC-DS](tpcds) and for +converting that data to Parquet format. Although it is valid to run benchmarks against CSV data, this does not really +represent how most of the world is running OLAP queries, especially when dealing with large datasets. When benchmarking +DataFusion and its subprojects, we typically want to be querying Parquet data. Also, we typically do not +want a single file per table, so we also need to repartition the data. The provided scripts take care of this conversion +and repartitioning. ## Running the Benchmarks with DataFusion @@ -67,10 +62,20 @@ Scripts are available for the following DataFusion projects: - [DataFusion Python](./runners/datafusion-python) - [DataFusion Comet](./runners/datafusion-comet) +These benchmarking scripts produce JSON files containing query timings. + ## Comparing Results -Coming soon. The plan is to add some Python scripts for comparing results from different runs and producing charts -that we can use in blog posts. +The Python script [scripts/generate-comparison.py](scripts/generate-comparison.py) can be used to produce charts +comparing results from different benchmark runs. + +For example: + +```shell +python scripts/generate-comparison.py file1.json file2.json --labels "Spark" "Comet" --benchmark "TPC-H 100GB" +``` + +This will create image files in the current directory in PNG format. ## Legal Notices diff --git a/scripts/generate-comparison.py b/scripts/generate-comparison.py new file mode 100644 index 0000000..b13904e --- /dev/null +++ b/scripts/generate-comparison.py @@ -0,0 +1,152 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import argparse +import json +import matplotlib.pyplot as plt +import numpy as np +import sys + +def geomean(data): + return np.prod(data) ** (1 / len(data)) + +def generate_query_speedup_chart(baseline, comparison, label1: str, label2: str, benchmark: str): + results = [] + for query in range(1, 23): + a = np.median(np.array(baseline[str(query)])) + b = np.median(np.array(comparison[str(query)])) + if a > b: + speedup = a/b-1 + else: + speedup = -(1/(a/b)-1) + results.append(("q" + str(query), round(speedup*100, 0))) + + results = sorted(results, key=lambda x: -x[1]) + + queries, speedups = zip(*results) + + # Create figure and axis + fig, ax = plt.subplots(figsize=(10, 6)) + + # Create bar chart + bars = ax.bar(queries, speedups, color='skyblue') + + # Add text annotations + for bar, speedup in zip(bars, speedups): + yval = bar.get_height() + if yval >= 0: + ax.text(bar.get_x() + bar.get_width() / 2.0, min(800, yval+5), f'{yval:.0f}%', va='bottom', ha='center', fontsize=8, + color='blue', rotation=90) + else: + ax.text(bar.get_x() + bar.get_width() / 2.0, yval, f'{yval:.0f}%', va='top', ha='center', fontsize=8, + color='blue', rotation=90) + + # Add title and labels + ax.set_title(label2 + " speedup over " + label1 + " (" + benchmark + ")") + ax.set_ylabel('Speedup (100% speedup = 2x faster)') + ax.set_xlabel('Query') + + # Customize the y-axis to handle both positive and negative values better + ax.axhline(0, color='black', linewidth=0.8) + min_value = (min(speedups) // 100) * 100 + max_value = ((max(speedups) // 100) + 1) * 100 + 50 + ax.set_ylim(min_value, max_value) + + # Show grid for better readability + ax.yaxis.grid(True) + + # Save the plot as an image file + plt.savefig('tpch_queries_speedup.png', format='png') + + +def generate_query_comparison_chart(results, labels, benchmark: str): + queries = [] + benches = [] + for _ in results: + benches.append([]) + for query in range(1, 23): + queries.append("q" + str(query)) + for i in range(0, len(results)): + benches[i].append(np.median(np.array(results[i][str(query)]))) + + # Define the width of the bars + bar_width = 0.3 + + # Define the positions of the bars on the x-axis + index = np.arange(len(queries)) * 1.5 + + # Create a bar chart + fig, ax = plt.subplots(figsize=(15, 6)) + for i in range(0, len(results)): + bar = ax.bar(index + i * bar_width, benches[i], bar_width, label=labels[i]) + + # Add labels, title, and legend + ax.set_title(benchmark) + ax.set_xlabel('Queries') + ax.set_ylabel('Query Time (seconds)') + ax.set_xticks(index + bar_width / 2) + ax.set_xticklabels(queries) + ax.legend() + + # Save the plot as an image file + plt.savefig('tpch_queries_compare.png', format='png') + +def generate_summary(results, labels, benchmark: str): + timings = [] + for _ in results: + timings.append(0) + + for query in range(1, 23): + for i in range(0, len(results)): + timings[i] += np.median(np.array(results[i][str(query)])) + + # Create figure and axis + fig, ax = plt.subplots() + + # Add title and labels + ax.set_title(benchmark) + ax.set_ylabel('Time in seconds to run all 22 TPC-H queries (lower is better)') + + times = [round(x,0) for x in timings] + + # Create bar chart + bars = ax.bar(labels, times, color='skyblue') + + # Add text annotations + for bar in bars: + yval = bar.get_height() + ax.text(bar.get_x() + bar.get_width() / 2.0, yval, f'{yval}', va='bottom') # va: vertical alignment + + plt.savefig('tpch_allqueries.png', format='png') + +def main(files, labels, benchmark: str): + results = [] + for filename in files: + with open(filename) as f: + results.append(json.load(f)) + generate_summary(results, labels, benchmark) + generate_query_comparison_chart(results, labels, benchmark) + if len(files) == 2: + generate_query_speedup_chart(results[0], results[1], labels[0], labels[1], benchmark) + +if __name__ == '__main__': + argparse = argparse.ArgumentParser(description='Generate comparison') + argparse.add_argument('filenames', nargs='+', type=str, help='JSON result files') + argparse.add_argument('--labels', nargs='+', type=str, help='Labels') + argparse.add_argument('--benchmark', type=str, help='Benchmark description') + args = argparse.parse_args() + main(args.filenames, args.labels, args.benchmark)