feat(tests): add TPC-H full test suite (#473)

* feat(tests): add TPC-H full test suite Signed-off-by: Alex Chi <iskyzh@gmail.com> * temporarily run tpch test in PR Signed-off-by: Alex Chi <iskyzh@gmail.com> * add tpch-test as ci task instead of bench task Signed-off-by: Alex Chi <iskyzh@gmail.com>
risinglightdb · Feb 15, 2022 · de55dfe · de55dfe
1 parent 3c6b867
commit de55dfe
Show file tree

Hide file tree

Showing 10 changed files with 270 additions and 24 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -44,3 +44,13 @@ jobs:
         with:
           command: test
           args: --no-fail-fast --all-features --locked
+
+  tpch-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+      - name: Run TPC-H test
+        run: ./tests/tpch-full.sh
diff --git a/docs/01-tpch.md b/docs/01-tpch.md
@@ -108,3 +108,49 @@ cargo run --release -- -f tests/sql/tpch/q10.sql
 
 All data of RisingLight is stored in `risinglight.secondary.db` folder. Simply remove it
 if you want to clean all data.
+
+## Developers: Add new TPC-H tests
+
+In `tests`, we have two kinds of TPC-H tests:
+
+* Run TPC-H query on small dataset (used for unit tests). `tpch` folder.
+* Run TPC-H query on ~1GB dataset generated by tpch-gen. `tpch-full` folder.
+
+Everytime we add a new TPC-H query, we should add both kinds of tests, in sqllogictest format. For example,
+developers should:
+
+* create `tests/sql/tpch/_qXX.slt`
+* create `tests/sql/tpch-full/_qXX.slt`
+* create `tests/sql/tpch/qXX.sql`
+* add `include _qXX.slt` in `tests/sql/tpch/tpch.slt`
+* add `include _qXX.slt` in `tests/sql/tpch-full/tpch.slt`
+
+By using `output-format` parameter, we can easily get the output format required by sqllogictest:
+
+```
+cargo run --release -- -f tests/sql/tpch/q5.sql --output-format text
+psql -d tpch -f tests/sql/tpch/q5.sql -A -t -F " "
+```
+
+... which yields
+
+```
+ALGERIA 55756674.2813
+MOZAMBIQUE 54883960.1257
+MOROCCO 50463646.0237
+ETHIOPIA 49934541.2268
+KENYA 48858086.8222
+```
+
+Developers should check the output against Postgres in order to ensure the output is correct. The full 1GB TPC-H
+test suite can be verified by running:
+
+```
+cargo run --release -- -f tests/sql/tpch-full/_tpch_full.slt --output-format text
+```
+
+And if you want to start fresh from empty database, you may run:
+
+```
+./tests/tpch-full.sh
+```
diff --git a/src/main.rs b/src/main.rs
@@ -26,29 +26,40 @@ struct Args {
     /// Whether to use in-memory engine
     #[clap(long)]
     memory: bool,
+
+    /// Control the output format
+    /// - `text`: plain text
+    /// - `human`: human readable format
+    #[clap(long)]
+    output_format: Option<String>,
 }
 
 // human-readable message
-fn print_chunk(chunk: &DataChunk) {
-    match chunk.header() {
-        Some(header) => match header[0].as_str() {
-            "$insert.row_counts" => {
-                println!("{} rows inserted", chunk.array_at(0).get_to_string(0))
-            }
-            "$delete.row_counts" => {
-                println!("{} rows deleted", chunk.array_at(0).get_to_string(0))
-            }
-            "$create" => println!("created"),
-            "$drop" => println!("dropped"),
-            "$explain" => println!("{}", chunk.array_at(0).get_to_string(0)),
-            _ => println!("{}", chunk),
+fn print_chunk(chunk: &DataChunk, output_format: &Option<String>) {
+    let output_format = output_format.as_ref().map(|x| x.as_str());
+    match output_format {
+        Some("human") | None => match chunk.header() {
+            Some(header) => match header[0].as_str() {
+                "$insert.row_counts" => {
+                    println!("{} rows inserted", chunk.array_at(0).get_to_string(0))
+                }
+                "$delete.row_counts" => {
+                    println!("{} rows deleted", chunk.array_at(0).get_to_string(0))
+                }
+                "$create" => println!("created"),
+                "$drop" => println!("dropped"),
+                "$explain" => println!("{}", chunk.array_at(0).get_to_string(0)),
+                _ => println!("{}", chunk),
+            },
+            None => println!("{}", chunk),
         },
-        None => println!("{}", chunk),
+        Some("text") => println!("{}", datachunk_to_sqllogictest_string(chunk)),
+        Some(format) => panic!("unsupported output format: {}", format),
     }
 }
 
 /// Run RisingLight interactive mode
-async fn interactive(db: Database) -> Result<()> {
+async fn interactive(db: Database, output_format: Option<String>) -> Result<()> {
     let mut rl = Editor::<()>::new();
     let history_path = dirs::cache_dir().map(|p| {
         let cache_dir = p.join("risinglight");
@@ -74,7 +85,7 @@ async fn interactive(db: Database) -> Result<()> {
                 match ret {
                     Ok(chunks) => {
                         for chunk in chunks {
-                            print_chunk(&chunk)
+                            print_chunk(&chunk, &output_format);
                         }
                     }
                     Err(err) => println!("{}", err),
@@ -104,13 +115,13 @@ async fn interactive(db: Database) -> Result<()> {
 }
 
 /// Run a SQL file in RisingLight
-async fn run_sql(db: Database, path: &str) -> Result<()> {
+async fn run_sql(db: Database, path: &str, output_format: Option<String>) -> Result<()> {
     let lines = std::fs::read_to_string(path)?;
 
     info!("{}", lines);
     let chunks = db.run(&lines).await?;
     for chunk in chunks {
-        print_chunk(&chunk)
+        print_chunk(&chunk, &output_format);
     }
 
     Ok(())
@@ -120,6 +131,7 @@ async fn run_sql(db: Database, path: &str) -> Result<()> {
 struct DatabaseWrapper {
     tx: tokio::sync::mpsc::Sender<String>,
     rx: Mutex<tokio::sync::mpsc::Receiver<Result<Vec<DataChunk>, risinglight::Error>>>,
+    output_format: Option<String>,
 }
 
 impl sqllogictest::DB for DatabaseWrapper {
@@ -129,7 +141,7 @@ impl sqllogictest::DB for DatabaseWrapper {
         self.tx.blocking_send(sql.to_string()).unwrap();
         let chunks = self.rx.lock().unwrap().blocking_recv().unwrap()?;
         for chunk in &chunks {
-            println!("{:?}", chunk);
+            print_chunk(chunk, &self.output_format);
         }
         let output = chunks
             .iter()
@@ -140,12 +152,13 @@ impl sqllogictest::DB for DatabaseWrapper {
 }
 
 /// Run a sqllogictest file in RisingLight
-async fn run_sqllogictest(db: Database, path: &str) -> Result<()> {
+async fn run_sqllogictest(db: Database, path: &str, output_format: Option<String>) -> Result<()> {
     let (ttx, mut trx) = tokio::sync::mpsc::channel(1);
     let (dtx, drx) = tokio::sync::mpsc::channel(1);
     let mut tester = sqllogictest::Runner::new(DatabaseWrapper {
         tx: ttx,
         rx: Mutex::new(drx),
+        output_format,
     });
     let handle = tokio::spawn(async move {
         while let Some(sql) = trx.recv().await {
@@ -188,15 +201,15 @@ async fn main() -> Result<()> {
 
     if let Some(file) = args.file {
         if file.ends_with(".sql") {
-            run_sql(db, &file).await?;
+            run_sql(db, &file, args.output_format).await?;
         } else if file.ends_with(".slt") {
-            run_sqllogictest(db, &file).await?;
+            run_sqllogictest(db, &file, args.output_format).await?;
         } else {
             warn!("No suffix detected, assume sql file");
-            run_sql(db, &file).await?;
+            run_sql(db, &file, args.output_format).await?;
         }
     } else {
-        interactive(db).await?;
+        interactive(db, args.output_format).await?;
     }
 
     Ok(())

diff --git a/tests/sql/tpch-full/_q1.slt b/tests/sql/tpch-full/_q1.slt
@@ -0,0 +1,27 @@
+query TTRRRRRRRI
+select
+    l_returnflag,
+    l_linestatus,
+    sum(l_quantity) as sum_qty,
+    sum(l_extendedprice) as sum_base_price,
+    sum(l_extendedprice * (1 - l_discount)) as sum_disc_price,
+    sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge,
+    avg(l_quantity) as avg_qty,
+    avg(l_extendedprice) as avg_price,
+    avg(l_discount) as avg_disc,
+    count(*) as count_order
+from
+    lineitem
+where
+    l_shipdate <= date '1998-12-01' - interval '71' day
+group by
+    l_returnflag,
+    l_linestatus
+order by
+    l_returnflag,
+    l_linestatus;
+----
+A F 37734107 56586554400.73 53758257134.8700 55909065222.827692 25.522005853257337031693758442 38273.129734621672202709109884 0.0499852958383976116221043995 1478493
+N F 991417 1487504710.38 1413082168.0541 1469649223.194375 25.516471920522983476604725382 38284.467760848303906933649045 0.0500934266742162969063674268 38854
+N O 75283683 112909876769.66 107265890064.7416 111560462484.162066 25.500840389078803497319290943 38245.960228243034695434326354 0.0499965957637073860749460403 2952204
+R F 37719753 56568041380.90 53741292684.6040 55889619119.831932 25.505793612690770655973817847 38250.85462609965717067761196 0.0500094058301270564687903602 1478870
diff --git a/tests/sql/tpch-full/_q10.slt b/tests/sql/tpch-full/_q10.slt
@@ -0,0 +1,54 @@
+query ITRRTTTT
+select
+    c_custkey,
+    c_name,
+    sum(l_extendedprice * (1 - l_discount)) as revenue,
+    c_acctbal,
+    n_name,
+    c_address,
+    c_phone,
+    c_comment
+from
+    customer,
+    orders,
+    lineitem,
+    nation
+where
+    c_custkey = o_custkey
+    and l_orderkey = o_orderkey
+    and o_orderdate >= date '1993-10-01'
+    and o_orderdate < date '1993-10-01' + interval '3' month
+    and l_returnflag = 'R'
+    and c_nationkey = n_nationkey
+group by
+    c_custkey,
+    c_name,
+    c_acctbal,
+    c_phone,
+    n_name,
+    c_address,
+    c_comment
+order by
+    revenue desc
+limit 20;
+----
+57040 Customer#000057040 734235.2455 632.87 JAPAN                     Eioyzjf4pp 22-895-641-3466 sits. slyly regular requests sleep alongside of the regular inst
+143347 Customer#000143347 721002.6948 2557.47 EGYPT                     1aReFYv,Kw4 14-742-935-3718 ggle carefully enticing requests. final deposits use bold, bold pinto beans. ironic, idle re
+60838 Customer#000060838 679127.3077 2454.77 BRAZIL                    64EaJ5vMAHWJlBOxJklpNc2RJiWE 12-913-494-9813  need to boost against the slyly regular account
+101998 Customer#000101998 637029.5667 3790.89 UNITED KINGDOM            01c9CILnNtfOQYmZj 33-593-865-6378 ress foxes wake slyly after the bold excuses. ironic platelets are furiously carefully bold theodolites
+125341 Customer#000125341 633508.0860 4983.51 GERMANY                   S29ODD6bceU8QSuuEJznkNaK 17-582-695-5962 arefully even depths. blithely even excuses sleep furiously. foxes use except the dependencies. ca
+25501 Customer#000025501 620269.7849 7725.04 ETHIOPIA                    W556MXuoiaYCCZamJI,Rn0B4ACUGdkQ8DZ 15-874-808-6793 he pending instructions wake carefully at the pinto beans. regular, final instructions along the slyly fina
+115831 Customer#000115831 596423.8672 5098.10 FRANCE                    rFeBbEEyk dl ne7zV5fDrmiq1oK09wV7pxqCgIc 16-715-386-3788 l somas sleep. furiously final deposits wake blithely regular pinto b
+84223 Customer#000084223 594998.0239 528.65 UNITED KINGDOM            nAVZCs6BaWap rrM27N 2qBnzc5WBauxbA 33-442-824-8191  slyly final deposits haggle regular, pending dependencies. pending escapades wake
+54289 Customer#000054289 585603.3918 5583.02 IRAN                      vXCxoCsU0Bad5JQI ,oobkZ 20-834-292-4707 ely special foxes are quickly finally ironic p
+39922 Customer#000039922 584878.1134 7321.11 GERMANY                   Zgy4s50l2GKN4pLDPBU8m342gIw6R 17-147-757-8036 y final requests. furiously final foxes cajole blithely special platelets. f
+6226 Customer#000006226 576783.7606 2230.09 UNITED KINGDOM            8gPu8,NPGkfyQQ0hcIYUGPIBWc,ybP5g, 33-657-701-3391 ending platelets along the express deposits cajole carefully final
+922 Customer#000000922 576767.5333 3869.25 GERMANY                   Az9RFaut7NkPnc5zSD2PwHgVwr4jRzq 17-945-916-9648 luffily fluffy deposits. packages c
+147946 Customer#000147946 576455.1320 2030.13 ALGERIA                   iANyZHjqhyy7Ajah0pTrYyhJ 10-886-956-3143 ithely ironic deposits haggle blithely ironic requests. quickly regu
+115640 Customer#000115640 569341.1933 6436.10 ARGENTINA                 Vtgfia9qI 7EpHgecU1X 11-411-543-4901 ost slyly along the patterns; pinto be
+73606 Customer#000073606 568656.8578 1785.67 JAPAN                     xuR0Tro5yChDfOCrjkd2ol 22-437-653-6966 he furiously regular ideas. slowly
+110246 Customer#000110246 566842.9815 7763.35 VIETNAM                   7KzflgX MDOq7sOkI 31-943-426-9837 egular deposits serve blithely above the fl
+142549 Customer#000142549 563537.2368 5085.99 INDONESIA                 ChqEoK43OysjdHbtKCp6dKqjNyvvi9 19-955-562-2398 sleep pending courts. ironic deposits against the carefully unusual platelets cajole carefully express accounts.
+146149 Customer#000146149 557254.9865 1791.55 ROMANIA                   s87fvzFQpU 29-744-164-6487  of the slyly silent accounts. quickly final accounts across the
+52528 Customer#000052528 556397.3509 551.79 ARGENTINA                 NFztyTOR10UOJ 11-208-192-3205  deposits hinder. blithely pending asymptotes breach slyly regular re
+23431 Customer#000023431 554269.5360 3381.86 ROMANIA                   HgiV0phqhaIa9aydNoIlb 29-915-458-2654 nusual, even instructions: furiously stealthy n
diff --git a/tests/sql/tpch-full/_q3.slt b/tests/sql/tpch-full/_q3.slt
@@ -0,0 +1,35 @@
+query IRTI
+select
+    l_orderkey,
+    sum(l_extendedprice * (1 - l_discount)) as revenue,
+    o_orderdate,
+    o_shippriority
+from
+    customer,
+    orders,
+    lineitem
+where
+    c_mktsegment = 'BUILDING'
+    and c_custkey = o_custkey
+    and l_orderkey = o_orderkey
+    and o_orderdate < date '1995-03-15'
+    and l_shipdate > date '1995-03-15'
+group by
+    l_orderkey,
+    o_orderdate,
+    o_shippriority
+order by
+    revenue desc,
+    o_orderdate
+limit 10;
+----
+2456423 406181.0111 1995-03-05 0
+3459808 405838.6989 1995-03-04 0
+492164 390324.0610 1995-02-19 0
+1188320 384537.9359 1995-03-09 0
+2435712 378673.0558 1995-02-26 0
+4878020 378376.7952 1995-03-12 0
+5521732 375153.9215 1995-03-13 0
+2628192 373133.3094 1995-02-22 0
+993600 371407.4595 1995-03-05 0
+2300070 367371.1452 1995-03-13 0
diff --git a/tests/sql/tpch-full/_q5.slt b/tests/sql/tpch-full/_q5.slt
@@ -0,0 +1,31 @@
+query TR
+select
+    n_name,
+    sum(l_extendedprice * (1 - l_discount)) as revenue
+from
+    customer,
+    orders,
+    lineitem,
+    supplier,
+    nation,
+    region
+where
+    c_custkey = o_custkey
+    and l_orderkey = o_orderkey
+    and l_suppkey = s_suppkey
+    and c_nationkey = s_nationkey
+    and s_nationkey = n_nationkey
+    and n_regionkey = r_regionkey
+    and r_name = 'AFRICA'
+    and o_orderdate >= date '1994-01-01'
+    and o_orderdate < date '1994-01-01' + interval '1' year
+group by
+    n_name
+order by
+    revenue desc;
+----
+ALGERIA 55756674.2813
+MOZAMBIQUE 54883960.1257
+MOROCCO 50463646.0237
+ETHIOPIA 49934541.2268
+KENYA 48858086.8222
diff --git a/tests/sql/tpch-full/_q6.slt b/tests/sql/tpch-full/_q6.slt
@@ -0,0 +1,12 @@
+query R
+select
+    sum(l_extendedprice * l_discount) as revenue
+from
+    lineitem
+where
+    l_shipdate >= date '1994-01-01'
+    and l_shipdate < date '1994-01-01' + interval '1' year
+    and l_discount between 0.08 - 0.01 and 0.08 + 0.01
+    and l_quantity < 24;
+----
+163821038.3323
diff --git a/tests/sql/tpch-full/_tpch_full.slt b/tests/sql/tpch-full/_tpch_full.slt
@@ -0,0 +1,5 @@
+include _q1.slt
+include _q3.slt
+include _q5.slt
+include _q6.slt
+include _q10.slt
diff --git a/tests/tpch-full.sh b/tests/tpch-full.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -xe
+
+rm -rf risinglight.secondary.db
+make tpch
+
+# TODO: run with SIMD feature
+
+cargo build --release
+cargo run --release -- -f tests/sql/tpch/create.sql
+cargo run --release -- -f tests/sql/tpch/import.sql
+cargo run --release -- -f tests/sql/tpch-full/_tpch_full.slt