chore: refactor options

config/frontend.example.toml

@@ -8,7 +8,7 @@ interval_millis = 5000
 retry_interval_millis = 5000
 
 # HTTP server options, see `standalone.example.toml`.
-[http_options]
+[http]
 addr = "127.0.0.1:4000"
 timeout = "30s"
 body_limit = "64MB"

src/cmd/Cargo.toml

@@ -38,12 +38,14 @@ datanode = { workspace = true }
 datatypes = { workspace = true }
 either = "1.8"
 etcd-client.workspace = true
+file-engine = { workspace = true }
 frontend = { workspace = true }
 futures.workspace = true
 lazy_static.workspace = true
 meta-client = { workspace = true }
 meta-srv = { workspace = true }
 metrics.workspace = true
+mito2 = { workspace = true }
 nu-ansi-term = "0.46"
 partition = { workspace = true }
 prost.workspace = true

src/cmd/src/frontend.rs

@@ -383,7 +383,7 @@ mod tests {
                     Some("11"),
                 ),
                 (
-                    // http_options.addr = 127.0.0.1:24000
+                    // http.addr = 127.0.0.1:24000
                     [
                         env_prefix.to_string(),
                         "http".to_uppercase(),

src/cmd/src/options.rs

@@ -29,11 +29,11 @@ pub const ENV_LIST_SEP: &str = ",";
 /// Options mixed up from datanode, frontend and metasrv.
 pub struct MixOptions {
     pub data_home: String,
-    pub procedure_cfg: ProcedureConfig,
-    pub kv_store_cfg: KvStoreConfig,
-    pub fe_opts: FrontendOptions,
-    pub dn_opts: DatanodeOptions,
-    pub logging_opts: LoggingOptions,
+    pub procedure: ProcedureConfig,
+    pub kv_store: KvStoreConfig,
+    pub frontend: FrontendOptions,
+    pub datanode: DatanodeOptions,
+    pub logging: LoggingOptions,
 }
 
 pub enum Options {
@@ -56,7 +56,7 @@ impl Options {
             Options::Datanode(opts) => &opts.logging,
             Options::Frontend(opts) => &opts.logging,
             Options::Metasrv(opts) => &opts.logging,
-            Options::Standalone(opts) => &opts.logging_opts,
+            Options::Standalone(opts) => &opts.logging,
             Options::Cli(opts) => opts,
         }
     }

src/cmd/src/standalone.rs

@@ -24,14 +24,16 @@ use common_meta::kv_backend::KvBackendRef;
 use common_procedure::ProcedureManagerRef;
 use common_telemetry::info;
 use common_telemetry::logging::LoggingOptions;
-use datanode::config::{DatanodeOptions, ProcedureConfig, StorageConfig};
+use datanode::config::{DatanodeOptions, ProcedureConfig, RegionEngineConfig, StorageConfig};
 use datanode::datanode::{Datanode, DatanodeBuilder};
 use datanode::region_server::RegionServer;
+use file_engine::config::EngineConfig as FileEngineConfig;
 use frontend::frontend::FrontendOptions;
 use frontend::instance::{FrontendInstance, Instance as FeInstance, StandaloneDatanodeManager};
 use frontend::service_config::{
     GrpcOptions, InfluxdbOptions, MysqlOptions, OpentsdbOptions, PostgresOptions, PromStoreOptions,
 };
+use mito2::config::MitoConfig;
 use serde::{Deserialize, Serialize};
 use servers::http::HttpOptions;
 use servers::tls::{TlsMode, TlsOption};
@@ -85,37 +87,43 @@ impl SubCommand {
 pub struct StandaloneOptions {
     pub mode: Mode,
     pub enable_telemetry: bool,
-    pub http_options: HttpOptions,
-    pub grpc_options: GrpcOptions,
-    pub mysql_options: MysqlOptions,
-    pub postgres_options: PostgresOptions,
-    pub opentsdb_options: OpentsdbOptions,
-    pub influxdb_options: InfluxdbOptions,
-    pub prom_store_options: PromStoreOptions,
+    pub http: HttpOptions,
+    pub grpc: GrpcOptions,
+    pub mysql: MysqlOptions,
+    pub postgres: PostgresOptions,
+    pub opentsdb: OpentsdbOptions,
+    pub influxdb: InfluxdbOptions,
+    pub prom_store: PromStoreOptions,
     pub wal: WalConfig,
     pub storage: StorageConfig,
     pub kv_store: KvStoreConfig,
     pub procedure: ProcedureConfig,
     pub logging: LoggingOptions,
+    /// Options for different store engines.
+    pub region_engine: Vec<RegionEngineConfig>,
 }
 
 impl Default for StandaloneOptions {
     fn default() -> Self {
         Self {
             mode: Mode::Standalone,
             enable_telemetry: true,
-            http_options: HttpOptions::default(),
-            grpc_options: GrpcOptions::default(),
-            mysql_options: MysqlOptions::default(),
-            postgres_options: PostgresOptions::default(),
-            opentsdb_options: OpentsdbOptions::default(),
-            influxdb_options: InfluxdbOptions::default(),
-            prom_store_options: PromStoreOptions::default(),
+            http: HttpOptions::default(),
+            grpc: GrpcOptions::default(),
+            mysql: MysqlOptions::default(),
+            postgres: PostgresOptions::default(),
+            opentsdb: OpentsdbOptions::default(),
+            influxdb: InfluxdbOptions::default(),
+            prom_store: PromStoreOptions::default(),
             wal: WalConfig::default(),
             storage: StorageConfig::default(),
             kv_store: KvStoreConfig::default(),
             procedure: ProcedureConfig::default(),
             logging: LoggingOptions::default(),
+            region_engine: vec![
+                RegionEngineConfig::Mito(MitoConfig::default()),
+                RegionEngineConfig::File(FileEngineConfig::default()),
+            ],
         }
     }
 }
@@ -124,13 +132,13 @@ impl StandaloneOptions {
     fn frontend_options(self) -> FrontendOptions {
         FrontendOptions {
             mode: self.mode,
-            http: self.http_options,
-            grpc: self.grpc_options,
-            mysql: self.mysql_options,
-            postgres: self.postgres_options,
-            opentsdb: self.opentsdb_options,
-            influxdb: self.influxdb_options,
-            prom_store: self.prom_store_options,
+            http: self.http,
+            grpc: self.grpc,
+            mysql: self.mysql,
+            postgres: self.postgres,
+            opentsdb: self.opentsdb,
+            influxdb: self.influxdb,
+            prom_store: self.prom_store,
             meta_client: None,
             logging: self.logging,
             ..Default::default()
@@ -143,6 +151,7 @@ impl StandaloneOptions {
             enable_telemetry: self.enable_telemetry,
             wal: self.wal,
             storage: self.storage,
+            region_engine: self.region_engine,
             ..Default::default()
         }
     }
@@ -232,7 +241,7 @@ impl StartCommand {
         );
 
         if let Some(addr) = &self.http_addr {
-            opts.http_options.addr = addr.clone()
+            opts.http.addr = addr.clone()
         }
 
         if let Some(addr) = &self.rpc_addr {
@@ -246,42 +255,42 @@ impl StartCommand {
                 }
                 .fail();
             }
-            opts.grpc_options.addr = addr.clone()
+            opts.grpc.addr = addr.clone()
         }
 
         if let Some(addr) = &self.mysql_addr {
-            opts.mysql_options.enable = true;
-            opts.mysql_options.addr = addr.clone();
-            opts.mysql_options.tls = tls_opts.clone();
+            opts.mysql.enable = true;
+            opts.mysql.addr = addr.clone();
+            opts.mysql.tls = tls_opts.clone();
         }
 
         if let Some(addr) = &self.postgres_addr {
-            opts.postgres_options.enable = true;
-            opts.postgres_options.addr = addr.clone();
-            opts.postgres_options.tls = tls_opts;
+            opts.postgres.enable = true;
+            opts.postgres.addr = addr.clone();
+            opts.postgres.tls = tls_opts;
         }
 
         if let Some(addr) = &self.opentsdb_addr {
-            opts.opentsdb_options.enable = true;
-            opts.opentsdb_options.addr = addr.clone();
+            opts.opentsdb.enable = true;
+            opts.opentsdb.addr = addr.clone();
         }
 
         if self.influxdb_enable {
-            opts.influxdb_options.enable = self.influxdb_enable;
+            opts.influxdb.enable = self.influxdb_enable;
         }
-        let kv_store_cfg = opts.kv_store.clone();
-        let procedure_cfg = opts.procedure.clone();
-        let fe_opts = opts.clone().frontend_options();
-        let logging_opts = opts.logging.clone();
-        let dn_opts = opts.datanode_options();
+        let kv_store = opts.kv_store.clone();
+        let procedure = opts.procedure.clone();
+        let frontend = opts.clone().frontend_options();
+        let logging = opts.logging.clone();
+        let datanode = opts.datanode_options();
 
         Ok(Options::Standalone(Box::new(MixOptions {
-            procedure_cfg,
-            kv_store_cfg,
-            data_home: dn_opts.storage.data_home.to_string(),
-            fe_opts,
-            dn_opts,
-            logging_opts,
+            procedure,
+            kv_store,
+            data_home: datanode.storage.data_home.to_string(),
+            frontend,
+            datanode,
+            logging,
         })))
     }
 
@@ -290,8 +299,8 @@ impl StartCommand {
     #[allow(clippy::diverging_sub_expression)]
     async fn build(self, opts: MixOptions) -> Result<Instance> {
         let plugins = Arc::new(load_frontend_plugins(&self.user_provider)?);
-        let fe_opts = opts.fe_opts;
-        let dn_opts = opts.dn_opts;
+        let fe_opts = opts.frontend;
+        let dn_opts = opts.datanode;
 
         info!("Standalone start command: {:#?}", self);
         info!(
@@ -300,13 +309,10 @@ impl StartCommand {
         );
 
         let kv_dir = kv_store_dir(&opts.data_home);
-        let (kv_store, procedure_manager) = FeInstance::try_build_standalone_components(
-            kv_dir,
-            opts.kv_store_cfg,
-            opts.procedure_cfg,
-        )
-        .await
-        .context(StartFrontendSnafu)?;
+        let (kv_store, procedure_manager) =
+            FeInstance::try_build_standalone_components(kv_dir, opts.kv_store, opts.procedure)
+                .await
+                .context(StartFrontendSnafu)?;
 
         let datanode =
             DatanodeBuilder::new(dn_opts.clone(), Some(kv_store.clone()), plugins.clone())
@@ -436,9 +442,9 @@ mod tests {
             checkpoint_margin = 9
             gc_duration = '7s'
 
-            [http_options]
+            [http]
             addr = "127.0.0.1:4000"
-            timeout = "30s"
+            timeout = "33s"
             body_limit = "128MB"
 
             [logging]
@@ -456,12 +462,12 @@ mod tests {
         else {
             unreachable!()
         };
-        let fe_opts = options.fe_opts;
-        let dn_opts = options.dn_opts;
-        let logging_opts = options.logging_opts;
+        let fe_opts = options.frontend;
+        let dn_opts = options.datanode;
+        let logging_opts = options.logging;
         assert_eq!(Mode::Standalone, fe_opts.mode);
         assert_eq!("127.0.0.1:4000".to_string(), fe_opts.http.addr);
-        assert_eq!(Duration::from_secs(30), fe_opts.http.timeout);
+        assert_eq!(Duration::from_secs(33), fe_opts.http.timeout);
         assert_eq!(ReadableSize::mb(128), fe_opts.http.body_limit);
         assert_eq!("127.0.0.1:4001".to_string(), fe_opts.grpc.addr);
         assert!(fe_opts.mysql.enable);
@@ -503,8 +509,8 @@ mod tests {
             unreachable!()
         };
 
-        assert_eq!("/tmp/greptimedb/test/logs", opts.logging_opts.dir);
-        assert_eq!("debug", opts.logging_opts.level.unwrap());
+        assert_eq!("/tmp/greptimedb/test/logs", opts.logging.dir);
+        assert_eq!("debug", opts.logging.level.unwrap());
     }
 
     #[test]
@@ -548,7 +554,7 @@ mod tests {
                     // http.addr = 127.0.0.1:24000
                     [
                         env_prefix.to_string(),
-                        "http_options".to_uppercase(),
+                        "http".to_uppercase(),
                         "addr".to_uppercase(),
                     ]
                     .join(ENV_VAR_SEP),
@@ -573,17 +579,17 @@ mod tests {
                 };
 
                 // Should be read from env, env > default values.
-                assert_eq!(opts.logging_opts.dir, "/other/log/dir");
+                assert_eq!(opts.logging.dir, "/other/log/dir");
 
                 // Should be read from config file, config file > env > default values.
-                assert_eq!(opts.logging_opts.level.as_ref().unwrap(), "debug");
+                assert_eq!(opts.logging.level.as_ref().unwrap(), "debug");
 
                 // Should be read from cli, cli > config file > env > default values.
-                assert_eq!(opts.fe_opts.http.addr, "127.0.0.1:14000");
-                assert_eq!(ReadableSize::mb(64), opts.fe_opts.http.body_limit);
+                assert_eq!(opts.frontend.http.addr, "127.0.0.1:14000");
+                assert_eq!(ReadableSize::mb(64), opts.frontend.http.body_limit);
 
                 // Should be default value.
-                assert_eq!(opts.fe_opts.grpc.addr, GrpcOptions::default().addr);
+                assert_eq!(opts.frontend.grpc.addr, GrpcOptions::default().addr);
             },
         );
     }

src/servers/tests/http/http_handler_test.rs

@@ -401,7 +401,7 @@ async fn test_config() {
     let toml_str = r#"
             mode = "distributed"
 
-            [http_options]
+            [http]
             addr = "127.0.0.1:4000"
             timeout = "30s"
             body_limit = "2GB"