Add script to generate comparison of two benchmark runs

.gitignore

@@ -1,2 +1,4 @@
+venv
 .idea
-*.json
+*.json
+*.png

README.md

@@ -48,17 +48,12 @@ prohibited by the TPC.
 
 ## Data Generation
 
-See the benchmark-specific instructions for generating the CSV data for [TPC-H](tpch) and [TPC-DS](tpcds).
-
-## Converting CSV data to Parquet
-
-Although it is valid to run benchmarks against CSV data, this does not really represent how most of the world is
-running OLAP queries, especially when dealing with large datasets. When benchmarking DataFusion, we typically want
-to be querying Parquet data, so first we must convert the generated datasets to Parquet. Also, we typically do not
-want a single file per table, so we also need to repartition the data.
-
-We plan on adding Python scripts in this repository to perform this conversion and repartitioning. Until then you may
-want to write your own scripts using DataFusion or Spark. Another option is to use [tpc-tools](https://crates.io/crates/tpctools).
+See the benchmark-specific instructions for generating the CSV data for [TPC-H](tpch) and [TPC-DS](tpcds) and for 
+converting that data to Parquet format. Although it is valid to run benchmarks against CSV data, this does not really 
+represent how most of the world is running OLAP queries, especially when dealing with large datasets. When benchmarking 
+DataFusion and its subprojects, we typically want to be querying Parquet data. Also, we typically do not
+want a single file per table, so we also need to repartition the data. The provided scripts take care of this conversion 
+and repartitioning.
 
 ## Running the Benchmarks with DataFusion
 
@@ -67,10 +62,20 @@ Scripts are available for the following DataFusion projects:
 - [DataFusion Python](./runners/datafusion-python)
 - [DataFusion Comet](./runners/datafusion-comet)
 
+These benchmarking scripts produce JSON files containing query timings.
+
 ## Comparing Results
 
-Coming soon. The plan is to add some Python scripts for comparing results from different runs and producing charts
-that we can use in blog posts.
+The Python script [scripts/generate-comparison.py](scripts/generate-comparison.py) can be used to produce charts 
+comparing results from different benchmark runs.
+
+For example:
+
+```shell
+python scripts/generate-comparison.py file1.json file2.json --labels "Spark" "Comet" --benchmark "TPC-H 100GB"
+```
+
+This will create image files in the current directory in PNG format.
 
 ## Legal Notices
 

scripts/generate-comparison.py

@@ -0,0 +1,152 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import json
+import matplotlib.pyplot as plt
+import numpy as np
+import sys
+
+def geomean(data):
+    return np.prod(data) ** (1 / len(data))
+
+def generate_query_speedup_chart(baseline, comparison, label1: str, label2: str, benchmark: str):
+    results = []
+    for query in range(1, 23):
+        a = np.median(np.array(baseline[str(query)]))
+        b = np.median(np.array(comparison[str(query)]))
+        if a > b:
+            speedup = a/b-1
+        else:
+            speedup = -(1/(a/b)-1)
+        results.append(("q" + str(query), round(speedup*100, 0)))
+
+    results = sorted(results, key=lambda x: -x[1])
+
+    queries, speedups = zip(*results)
+
+    # Create figure and axis
+    fig, ax = plt.subplots(figsize=(10, 6))
+
+    # Create bar chart
+    bars = ax.bar(queries, speedups, color='skyblue')
+
+    # Add text annotations
+    for bar, speedup in zip(bars, speedups):
+        yval = bar.get_height()
+        if yval >= 0:
+            ax.text(bar.get_x() + bar.get_width() / 2.0, min(800, yval+5), f'{yval:.0f}%', va='bottom', ha='center', fontsize=8,
+                    color='blue', rotation=90)
+        else:
+            ax.text(bar.get_x() + bar.get_width() / 2.0, yval, f'{yval:.0f}%', va='top', ha='center', fontsize=8,
+                    color='blue', rotation=90)
+
+    # Add title and labels
+    ax.set_title(label2 + " speedup over " + label1 + " (" + benchmark + ")")
+    ax.set_ylabel('Speedup (100% speedup = 2x faster)')
+    ax.set_xlabel('Query')
+
+    # Customize the y-axis to handle both positive and negative values better
+    ax.axhline(0, color='black', linewidth=0.8)
+    min_value = (min(speedups) // 100) * 100
+    max_value = ((max(speedups) // 100) + 1) * 100 + 50
+    ax.set_ylim(min_value, max_value)
+
+    # Show grid for better readability
+    ax.yaxis.grid(True)
+
+    # Save the plot as an image file
+    plt.savefig('tpch_queries_speedup.png', format='png')
+
+
+def generate_query_comparison_chart(results, labels, benchmark: str):
+    queries = []
+    benches = []
+    for _ in results:
+        benches.append([])
+    for query in range(1, 23):
+        queries.append("q" + str(query))
+        for i in range(0, len(results)):
+            benches[i].append(np.median(np.array(results[i][str(query)])))
+
+    # Define the width of the bars
+    bar_width = 0.3
+
+    # Define the positions of the bars on the x-axis
+    index = np.arange(len(queries)) * 1.5
+
+    # Create a bar chart
+    fig, ax = plt.subplots(figsize=(15, 6))
+    for i in range(0, len(results)):
+        bar = ax.bar(index + i * bar_width, benches[i], bar_width, label=labels[i])
+
+    # Add labels, title, and legend
+    ax.set_title(benchmark)
+    ax.set_xlabel('Queries')
+    ax.set_ylabel('Query Time (seconds)')
+    ax.set_xticks(index + bar_width / 2)
+    ax.set_xticklabels(queries)
+    ax.legend()
+
+    # Save the plot as an image file
+    plt.savefig('tpch_queries_compare.png', format='png')
+
+def generate_summary(results, labels, benchmark: str):
+    timings = []
+    for _ in results:
+        timings.append(0)
+
+    for query in range(1, 23):
+        for i in range(0, len(results)):
+            timings[i] += np.median(np.array(results[i][str(query)]))
+
+    # Create figure and axis
+    fig, ax = plt.subplots()
+
+    # Add title and labels
+    ax.set_title(benchmark)
+    ax.set_ylabel('Time in seconds to run all 22 TPC-H queries (lower is better)')
+
+    times = [round(x,0) for x in timings]
+
+    # Create bar chart
+    bars = ax.bar(labels, times, color='skyblue')
+
+    # Add text annotations
+    for bar in bars:
+        yval = bar.get_height()
+        ax.text(bar.get_x() + bar.get_width() / 2.0, yval, f'{yval}', va='bottom')  # va: vertical alignment
+
+    plt.savefig('tpch_allqueries.png', format='png')
+
+def main(files, labels, benchmark: str):
+    results = []
+    for filename in files:
+        with open(filename) as f:
+            results.append(json.load(f))
+    generate_summary(results, labels, benchmark)
+    generate_query_comparison_chart(results, labels, benchmark)
+    if len(files) == 2:
+        generate_query_speedup_chart(results[0], results[1], labels[0], labels[1], benchmark)
+
+if __name__ == '__main__':
+    argparse = argparse.ArgumentParser(description='Generate comparison')
+    argparse.add_argument('filenames', nargs='+', type=str, help='JSON result files')
+    argparse.add_argument('--labels', nargs='+', type=str, help='Labels')
+    argparse.add_argument('--benchmark', type=str, help='Benchmark description')
+    args = argparse.parse_args()
+    main(args.filenames, args.labels, args.benchmark)