Bump datafusion to 36.0.0 and make ballista compatible with it. (#1027)

Cargo.toml

@@ -35,11 +35,11 @@ arrow-flight = { version = "50.0.0", features = ["flight-sql-experimental"] }
 arrow-schema = { version = "50.0.0", default-features = false }
 configure_me = { version = "0.4.0" }
 configure_me_codegen = { version = "0.4.4" }
-datafusion = "35.0.0"
-datafusion-cli = "35.0.0"
-datafusion-proto = "35.0.0"
+datafusion = "36.0.0"
+datafusion-cli = "36.0.0"
+datafusion-proto = "36.0.0"
 object_store = "0.9.0"
-sqlparser = "0.41.0"
+sqlparser = "0.43.0"
 tonic = { version = "0.10" }
 tonic-build = { version = "0.10", default-features = false, features = [
     "transport",

ballista-cli/README.md

@@ -32,6 +32,9 @@ OPTIONS:
     -c, --batch-size <BATCH_SIZE>
             The batch size of each query, or use Ballista default
 
+        --color
+            Enables console syntax highlighting
+
         --concurrent-tasks <CONCURRENT_TASKS>
             The max concurrent tasks, only for Ballista local mode. Default: all available cores
 

ballista-cli/src/exec.rs

@@ -95,7 +95,15 @@ pub async fn exec_from_files(
 /// run and execute SQL statements and commands against a context with the given print options
 pub async fn exec_from_repl(ctx: &BallistaContext, print_options: &mut PrintOptions) {
     let mut rl = Editor::new().expect("created editor");
-    rl.set_helper(Some(CliHelper::default()));
+    rl.set_helper(Some(CliHelper::new(
+        &ctx.context()
+            .task_ctx()
+            .session_config()
+            .options()
+            .sql_parser
+            .dialect,
+        print_options.color,
+    )));
     rl.load_history(".history").ok();
 
     let mut print_options = print_options.clone();

ballista-cli/src/main.rs

@@ -89,6 +89,9 @@ struct Args {
         help = "Reduce printing other than the results and work quietly"
     )]
     quiet: bool,
+
+    #[clap(long, help = "Enables console syntax highlighting")]
+    color: bool,
 }
 
 #[tokio::main]
@@ -135,6 +138,7 @@ pub async fn main() -> Result<()> {
         format: args.format,
         quiet: args.quiet,
         maxrows: MaxRows::Unlimited,
+        color: args.color,
     };
 
     let files = args.file;

ballista/client/src/context.rs

@@ -345,6 +345,10 @@ impl BallistaContext {
         Ok(is_show_variable)
     }
 
+    pub fn context(&self) -> &SessionContext {
+        &self.context
+    }
+
     /// Create a DataFrame from a SQL statement.
     ///
     /// This method is `async` because queries of type `CREATE EXTERNAL TABLE`