Improve sqllogicteset speed by creating only a single large file rather than 2

datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs

@@ -2313,6 +2313,8 @@ fn simplify_right_is_one_case(
     }
 }
 
+// trigger ci test
+
 #[cfg(test)]
 mod tests {
     use super::*;

datafusion/sqllogictest/bin/sqllogictests.rs

@@ -18,7 +18,9 @@
 use clap::{ColorChoice, Parser};
 use datafusion::common::instant::Instant;
 use datafusion::common::utils::get_available_parallelism;
-use datafusion::common::{DataFusionError, Result, exec_datafusion_err, exec_err};
+use datafusion::common::{
+    DataFusionError, HashMap, Result, exec_datafusion_err, exec_err,
+};
 use datafusion_sqllogictest::{
     CurrentlyExecutingSqlTracker, DataFusion, DataFusionSubstraitRoundTrip, Filter,
     TestContext, df_value_validator, read_dir_recursive, setup_scratch_dir,
@@ -47,8 +49,8 @@ use std::fs;
 use std::io::{IsTerminal, stderr, stdout};
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
-use std::sync::Arc;
 use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::{Arc, LazyLock};
 
 #[cfg(feature = "postgres")]
 mod postgres_container;
@@ -59,6 +61,31 @@ const PG_COMPAT_FILE_PREFIX: &str = "pg_compat_";
 const SQLITE_PREFIX: &str = "sqlite";
 const ERRS_PER_FILE_LIMIT: usize = 10;
 
+/// TEST PRIORITY
+///
+/// Heuristically prioritize some test to run earlier.
+///
+/// Prioritizes test to run earlier if they are known to be long running (as
+/// each test file itself is run sequentially, but multiple test files are run
+/// in parallel.
+///
+/// Tests not listed here will run after the listed tests in an arbitrary order.
+static TEST_PRIORITY: LazyLock<HashMap<PathBuf, usize>> = LazyLock::new(|| {
+    [
+        (PathBuf::from("push_down_filter_regression.slt"), 0), // longest running, so run first.
+        (PathBuf::from(" aggregate.slt"), 1),
+        (PathBuf::from(" joins.slt"), 2),
+        (PathBuf::from("imdb.slt"), 3),
+        (PathBuf::from("array.slt"), 4),
+    ]
+    .into_iter()
+    .collect()
+});
+
+/// Default priority for tests not in the TEST_PRIORITY map. Tests with lower
+/// priority values run first.
+static DEFAULT_PRIORITY: usize = 100;
+
 pub fn main() -> Result<()> {
     tokio::runtime::Builder::new_multi_thread()
         .enable_all()
@@ -739,7 +766,21 @@ fn read_test_files(options: &Options) -> Result<Vec<TestFile>> {
         paths.append(&mut sqlite_paths)
     }
 
-    Ok(paths)
+    Ok(sort_tests(paths))
+}
+
+/// Sort the tests heuristically by order of "priority"
+///
+/// Prioritizes test to run earlier if they are known to be long running (as
+/// each test file itself is run sequentially, but multiple test files are run
+/// in parallel.
+fn sort_tests(mut tests: Vec<TestFile>) -> Vec<TestFile> {
+    tests.sort_by_key(|f| {
+        TEST_PRIORITY
+            .get(&f.relative_path)
+            .unwrap_or(&DEFAULT_PRIORITY)
+    });
+    tests
 }
 
 /// Parsed command line options