Add CI check to verify that benchmark queries return the expected results

.github/workflows/rust.yml

@@ -117,6 +117,47 @@ jobs:
       - name: Verify Working Directory Clean
         run: git diff --exit-code
 
+  # verify that the benchmark queries return the correct results
+  verify-benchmark-results:
+    name: verify benchmark results (amd64)
+    needs: [linux-build-lib]
+    runs-on: ubuntu-latest
+    container:
+      image: amd64/rust
+      env:
+        # Disable full debug symbol generation to speed up CI build and keep memory down
+        # "1" means line tables only, which is useful for panic tracebacks.
+        RUSTFLAGS: "-C debuginfo=1"
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Cache Cargo
+        uses: actions/cache@v3
+        with:
+          path: /github/home/.cargo
+          # this key equals the ones on `linux-build-lib` for re-use
+          key: cargo-cache-
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: stable
+      - name: Generate benchmark data and expected query results
+        run: |
+          mkdir -p benchmarks/data/answers
+          git clone https://github.com/databricks/tpch-dbgen.git
+          cd tpch-dbgen
+          make  
+          ./dbgen -f -s 1
+          mv *.tbl ../benchmarks/data
+          mv ./answers/* ../benchmarks/data/answers/
+      - name: Verify that benchmark queries return expected results
+        run: |
+          export TPCH_DATA=`pwd`/benchmarks/data
+          cargo test verify_q --profile release-nonlto --features=ci -- --test-threads=1
+      - name: Verify Working Directory Clean
+        run: git diff --exit-code
+
   integration-test:
     name: "Compare to postgres"
     needs: [linux-build-lib]

Cargo.toml

@@ -16,21 +16,24 @@
 # under the License.
 
 [workspace]
-members = [
-    "datafusion/common",
-    "datafusion/core",
-    "datafusion/expr",
-    "datafusion/jit",
-    "datafusion/optimizer",
-    "datafusion/physical-expr",
-    "datafusion/proto",
-    "datafusion/row",
-    "datafusion/sql",
-    "datafusion-examples",
-    "benchmarks",
-]
 exclude = ["datafusion-cli"]
+members = ["datafusion/common", "datafusion/core", "datafusion/expr", "datafusion/jit", "datafusion/optimizer", "datafusion/physical-expr", "datafusion/proto", "datafusion/row", "datafusion/sql", "datafusion-examples", "benchmarks",
+]
 
 [profile.release]
 codegen-units = 1
 lto = true
+
+# the release profile takes a long time to build so we can use this profile during development to save time
+# cargo build --profile release-nonlto
+[profile.release-nonlto]
+codegen-units = 16
+debug = false
+debug-assertions = false
+incremental = false
+inherits = "release"
+lto = false
+opt-level = 3
+overflow-checks = false
+panic = 'unwind'
+rpath = false

benchmarks/Cargo.toml

@@ -24,10 +24,10 @@ authors = ["Apache Arrow <dev@arrow.apache.org>"]
 homepage = "https://github.com/apache/arrow-datafusion"
 repository = "https://github.com/apache/arrow-datafusion"
 license = "Apache-2.0"
-publish = false
 rust-version = "1.62"
 
 [features]
+ci = []
 default = ["mimalloc"]
 simd = ["datafusion/simd"]
 snmalloc = ["snmalloc-rs"]