diff --git a/risedev.yml b/risedev.yml index 7c57eb160166..1e3d10673677 100644 --- a/risedev.yml +++ b/risedev.yml @@ -105,6 +105,7 @@ profile: - use: minio - use: etcd - use: meta-node + meta-backend: etcd - use: compute-node - use: frontend - use: compactor @@ -119,6 +120,7 @@ profile: - use: etcd - use: meta-node user-managed: true + meta-backend: etcd - use: compute-node user-managed: true - use: frontend @@ -136,6 +138,7 @@ profile: - use: etcd - use: meta-node user-managed: true + meta-backend: etcd - use: compute-node user-managed: true - use: frontend @@ -149,6 +152,7 @@ profile: - use: etcd - use: meta-node user-managed: true + meta-backend: etcd - use: compute-node user-managed: true - use: frontend @@ -253,6 +257,7 @@ profile: - use: minio - use: etcd - use: meta-node + meta-backend: etcd - use: compute-node - use: frontend - use: compactor @@ -286,6 +291,7 @@ profile: port: 5690 dashboard-port: 5691 exporter-port: 1250 + meta-backend: etcd - use: meta-node port: 15690 dashboard-port: 15691 @@ -333,6 +339,7 @@ profile: port: 5690 dashboard-port: 5691 exporter-port: 1250 + meta-backend: etcd - use: meta-node port: 15690 dashboard-port: 15691 @@ -353,6 +360,7 @@ profile: port: 5690 dashboard-port: 5691 exporter-port: 1250 + meta-backend: sqlite - use: compactor - use: compute-node - use: frontend @@ -366,6 +374,40 @@ profile: port: 5690 dashboard-port: 5691 exporter-port: 1250 + meta-backend: sqlite + - use: compactor + - use: compute-node + - use: frontend + + meta-1cn-1fe-pg-backend: + steps: + - use: minio + - use: postgres + port: 8432 + user: postgres + database: metadata + - use: meta-node + port: 5690 + dashboard-port: 5691 + exporter-port: 1250 + meta-backend: postgres + - use: compactor + - use: compute-node + - use: frontend + + meta-1cn-1fe-pg-backend-with-recovery: + config-path: src/config/ci-recovery.toml + steps: + - use: minio + - use: postgres + port: 8432 + user: postgres + database: metadata + - use: meta-node + port: 5690 + dashboard-port: 5691 + exporter-port: 1250 + meta-backend: postgres - use: compactor - use: compute-node - use: frontend @@ -392,6 +434,7 @@ profile: - use: minio - use: etcd - use: meta-node + meta-backend: etcd - use: compute-node parallelism: 8 - use: frontend @@ -550,6 +593,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node enable-tiered-cache: true - use: frontend @@ -562,6 +606,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node port: 5687 exporter-port: 1222 @@ -584,6 +629,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node port: 5687 exporter-port: 1222 @@ -610,6 +656,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node port: 5687 exporter-port: 1222 @@ -634,6 +681,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node port: 5687 exporter-port: 1222 @@ -658,6 +706,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node port: 5687 exporter-port: 1222 @@ -717,6 +766,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: opendal engine: fs bucket: "/tmp/rw_ci" @@ -750,6 +800,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node port: 5687 exporter-port: 1222 @@ -801,6 +852,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node enable-tiered-cache: true - use: frontend @@ -817,6 +869,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node enable-tiered-cache: true - use: frontend @@ -841,6 +894,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node enable-tiered-cache: true - use: frontend @@ -854,6 +908,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node enable-tiered-cache: true total-memory-bytes: 17179869184 @@ -866,6 +921,7 @@ profile: - use: minio - use: etcd - use: meta-node + meta-backend: etcd - use: compute-node enable-tiered-cache: true - use: frontend @@ -878,6 +934,7 @@ profile: - use: etcd unsafe-no-fsync: true - use: meta-node + meta-backend: etcd - use: compute-node port: 5687 exporter-port: 1222 @@ -899,6 +956,7 @@ profile: - use: minio - use: etcd - use: meta-node + meta-backend: etcd - use: compute-node enable-tiered-cache: true - use: frontend @@ -912,6 +970,7 @@ profile: - use: etcd - use: minio - use: meta-node + meta-backend: etcd - use: compute-node - use: frontend - use: compactor @@ -928,6 +987,7 @@ profile: - use: sqlite - use: minio - use: meta-node + meta-backend: sqlite - use: compute-node - use: frontend - use: compactor @@ -978,6 +1038,7 @@ profile: - use: minio - use: etcd - use: meta-node + meta-backend: etcd - use: compute-node - use: frontend - use: compactor @@ -989,6 +1050,7 @@ profile: bucket: renjie-iceberg-bench - use: etcd - use: meta-node + meta-backend: etcd - use: compute-node - use: frontend - use: compactor @@ -1001,6 +1063,7 @@ profile: - use: minio - use: etcd - use: meta-node + meta-backend: etcd - use: compute-node - use: frontend - use: compactor @@ -1161,12 +1224,18 @@ template: # If `user-managed` is true, this service will be started by user with the above config user-managed: false + # meta backend type, requires extra config for provided backend + meta-backend: "memory" + # Etcd backend config provide-etcd-backend: "etcd*" # Sqlite backend config provide-sqlite-backend: "sqlite*" + # Postgres backend config + provide-postgres-backend: "postgres*" + # Prometheus nodes used by dashboard service provide-prometheus: "prometheus*" @@ -1444,3 +1513,32 @@ template: # If `user-managed` is true, user is responsible for starting the service # to serve at the above address and port in any way they see fit. user-managed: false + + # PostgreSQL service backed by docker. + postgres: + # Id to be picked-up by services + id: postgres-${port} + + # address of pg + address: "127.0.0.1" + + # listen port of pg + port: 8432 + + # Note: + # - This will be used to initialize the PostgreSQL instance if it's fresh. + # - In user-managed mode, these configs are not validated by risedev. + # They are passed as-is to risedev-env default user for PostgreSQL operations. + user: postgres + password: "" + database: "postgres" + + # The docker image. Can be overridden to use a different version. + image: "postgres:15-alpine" + + # If set to true, data will be persisted at data/{id}. + persist-data: true + + # If `user-managed` is true, user is responsible for starting the service + # to serve at the above address and port in any way they see fit. + user-managed: false diff --git a/src/cmd_all/src/single_node.rs b/src/cmd_all/src/single_node.rs index 63099c1ff640..2340841a13c3 100644 --- a/src/cmd_all/src/single_node.rs +++ b/src/cmd_all/src/single_node.rs @@ -186,7 +186,7 @@ pub fn map_single_node_opts_to_standalone_opts(opts: SingleNodeOpts) -> ParsedSt std::fs::create_dir_all(&meta_store_dir).unwrap(); let meta_store_endpoint = format!("sqlite://{}/single_node.db?mode=rwc", &meta_store_dir); - meta_opts.sql_endpoint = Some(meta_store_endpoint); + meta_opts.sql_endpoint = Some(meta_store_endpoint.into()); } } diff --git a/src/meta/node/src/lib.rs b/src/meta/node/src/lib.rs index 140cf481e3c4..c7d8e95d7e49 100644 --- a/src/meta/node/src/lib.rs +++ b/src/meta/node/src/lib.rs @@ -76,7 +76,7 @@ pub struct MetaNodeOpts { /// Endpoint of the SQL service, make it non-option when SQL service is required. #[clap(long, hide = true, env = "RW_SQL_ENDPOINT")] - pub sql_endpoint: Option, + pub sql_endpoint: Option>, /// The HTTP REST-API address of the Prometheus instance associated to this cluster. /// This address is used to serve `PromQL` queries to Prometheus. @@ -221,7 +221,11 @@ pub fn start(opts: MetaNodeOpts) -> Pin + Send>> { }, MetaBackend::Mem => MetaStoreBackend::Mem, MetaBackend::Sql => MetaStoreBackend::Sql { - endpoint: opts.sql_endpoint.expect("sql endpoint is required"), + endpoint: opts + .sql_endpoint + .expect("sql endpoint is required") + .expose_secret() + .to_string(), }, }; diff --git a/src/risedevtool/src/bin/risedev-compose.rs b/src/risedevtool/src/bin/risedev-compose.rs index 10b29e836c66..748da07a9d35 100644 --- a/src/risedevtool/src/bin/risedev-compose.rs +++ b/src/risedevtool/src/bin/risedev-compose.rs @@ -219,7 +219,7 @@ fn main() -> Result<()> { volumes.insert(c.id.clone(), ComposeVolume::default()); (c.address.clone(), c.compose(&compose_config)?) } - ServiceConfig::Redis(_) | ServiceConfig::MySql(_) => { + ServiceConfig::Redis(_) | ServiceConfig::MySql(_) | ServiceConfig::Postgres(_) => { return Err(anyhow!("not supported")) } }; diff --git a/src/risedevtool/src/bin/risedev-dev.rs b/src/risedevtool/src/bin/risedev-dev.rs index f11bc1b3b514..4f636cb7e30b 100644 --- a/src/risedevtool/src/bin/risedev-dev.rs +++ b/src/risedevtool/src/bin/risedev-dev.rs @@ -26,8 +26,9 @@ use risedev::util::{complete_spin, fail_spin}; use risedev::{ generate_risedev_env, preflight_check, CompactorService, ComputeNodeService, ConfigExpander, ConfigureTmuxTask, DummyService, EnsureStopService, ExecuteContext, FrontendService, - GrafanaService, KafkaService, MetaNodeService, MinioService, MySqlService, PrometheusService, - PubsubService, RedisService, ServiceConfig, SqliteConfig, Task, TempoService, RISEDEV_NAME, + GrafanaService, KafkaService, MetaNodeService, MinioService, MySqlService, PostgresService, + PrometheusService, PubsubService, RedisService, ServiceConfig, SqliteConfig, Task, + TempoService, RISEDEV_NAME, }; use tempfile::tempdir; use thiserror_ext::AsReport; @@ -314,6 +315,16 @@ fn task_main( ctx.pb .set_message(format!("mysql {}:{}", c.address, c.port)); } + ServiceConfig::Postgres(c) => { + let mut ctx = + ExecuteContext::new(&mut logger, manager.new_progress(), status_dir.clone()); + PostgresService::new(c.clone()).execute(&mut ctx)?; + let mut task = + risedev::ConfigureTcpNodeTask::new(c.address.clone(), c.port, c.user_managed)?; + task.execute(&mut ctx)?; + ctx.pb + .set_message(format!("postgres {}:{}", c.address, c.port)); + } } let service_id = service.id().to_string(); diff --git a/src/risedevtool/src/config.rs b/src/risedevtool/src/config.rs index 541f269ee18c..839ebc22486e 100644 --- a/src/risedevtool/src/config.rs +++ b/src/risedevtool/src/config.rs @@ -174,6 +174,7 @@ impl ConfigExpander { "redis" => ServiceConfig::Redis(serde_yaml::from_str(&out_str)?), "redpanda" => ServiceConfig::RedPanda(serde_yaml::from_str(&out_str)?), "mysql" => ServiceConfig::MySql(serde_yaml::from_str(&out_str)?), + "postgres" => ServiceConfig::Postgres(serde_yaml::from_str(&out_str)?), other => return Err(anyhow!("unsupported use type: {}", other)), }; Ok(result) diff --git a/src/risedevtool/src/risedev_env.rs b/src/risedevtool/src/risedev_env.rs index 729279328e93..b33a65f9986a 100644 --- a/src/risedevtool/src/risedev_env.rs +++ b/src/risedevtool/src/risedev_env.rs @@ -96,6 +96,17 @@ pub fn generate_risedev_env(services: &Vec) -> String { let port = &c.port; writeln!(env, r#"RISEDEV_PUBSUB_WITH_OPTIONS_COMMON="connector='google_pubsub',pubsub.emulator_host='{address}:{port}'""#,).unwrap(); } + ServiceConfig::Postgres(c) => { + let host = &c.address; + let port = &c.port; + let user = &c.user; + let password = &c.password; + // These envs are used by `postgres` cli. + writeln!(env, r#"PGHOST="{host}""#,).unwrap(); + writeln!(env, r#"PGPORT="{port}""#,).unwrap(); + writeln!(env, r#"PGUSER="{user}""#,).unwrap(); + writeln!(env, r#"PGPASSWORD="{password}""#,).unwrap(); + } _ => {} } } diff --git a/src/risedevtool/src/service_config.rs b/src/risedevtool/src/service_config.rs index 3135d7af4c00..4811ef96f5b8 100644 --- a/src/risedevtool/src/service_config.rs +++ b/src/risedevtool/src/service_config.rs @@ -60,8 +60,10 @@ pub struct MetaNodeConfig { pub user_managed: bool, + pub meta_backend: String, pub provide_etcd_backend: Option>, pub provide_sqlite_backend: Option>, + pub provide_postgres_backend: Option>, pub provide_prometheus: Option>, pub provide_compute_node: Option>, @@ -341,6 +343,26 @@ pub struct MySqlConfig { pub persist_data: bool, } +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +#[serde(deny_unknown_fields)] +pub struct PostgresConfig { + #[serde(rename = "use")] + phantom_use: Option, + pub id: String, + + pub port: u16, + pub address: String, + + pub user: String, + pub password: String, + pub database: String, + + pub image: String, + pub user_managed: bool, + pub persist_data: bool, +} + /// All service configuration #[derive(Clone, Debug, PartialEq)] pub enum ServiceConfig { @@ -361,6 +383,7 @@ pub enum ServiceConfig { Redis(RedisConfig), RedPanda(RedPandaConfig), MySql(MySqlConfig), + Postgres(PostgresConfig), } impl ServiceConfig { @@ -383,6 +406,7 @@ impl ServiceConfig { Self::RedPanda(c) => &c.id, Self::Opendal(c) => &c.id, Self::MySql(c) => &c.id, + ServiceConfig::Postgres(c) => &c.id, } } @@ -405,6 +429,7 @@ impl ServiceConfig { Self::RedPanda(_c) => None, Self::Opendal(_) => None, Self::MySql(c) => Some(c.port), + ServiceConfig::Postgres(c) => Some(c.port), } } @@ -427,6 +452,7 @@ impl ServiceConfig { Self::RedPanda(_c) => false, Self::Opendal(_c) => false, Self::MySql(c) => c.user_managed, + Self::Postgres(c) => c.user_managed, } } } diff --git a/src/risedevtool/src/task.rs b/src/risedevtool/src/task.rs index 24474d14c600..fbabc6a27d22 100644 --- a/src/risedevtool/src/task.rs +++ b/src/risedevtool/src/task.rs @@ -25,6 +25,7 @@ mod kafka_service; mod meta_node_service; mod minio_service; mod mysql_service; +mod postgres_service; mod prometheus_service; mod pubsub_service; mod redis_service; @@ -62,6 +63,7 @@ pub use self::kafka_service::*; pub use self::meta_node_service::*; pub use self::minio_service::*; pub use self::mysql_service::*; +pub use self::postgres_service::*; pub use self::prometheus_service::*; pub use self::pubsub_service::*; pub use self::redis_service::*; diff --git a/src/risedevtool/src/task/meta_node_service.rs b/src/risedevtool/src/task/meta_node_service.rs index 42d555f48b3d..a1bbd2c04e6c 100644 --- a/src/risedevtool/src/task/meta_node_service.rs +++ b/src/risedevtool/src/task/meta_node_service.rs @@ -77,35 +77,58 @@ impl MetaNodeService { let mut is_persistent_meta_store = false; - if let Some(sqlite_config) = &config.provide_sqlite_backend - && !sqlite_config.is_empty() - { - is_persistent_meta_store = true; - let prefix_data = env::var("PREFIX_DATA")?; - let file_path = PathBuf::from(&prefix_data) - .join(&sqlite_config[0].id) - .join(&sqlite_config[0].file); - cmd.arg("--backend") - .arg("sql") - .arg("--sql-endpoint") - .arg(format!("sqlite://{}?mode=rwc", file_path.display())); - } else { - match config.provide_etcd_backend.as_ref().unwrap().as_slice() { - [] => { - cmd.arg("--backend").arg("mem"); - } - etcds => { - is_persistent_meta_store = true; - cmd.arg("--backend") - .arg("etcd") - .arg("--etcd-endpoints") - .arg( - etcds - .iter() - .map(|etcd| format!("{}:{}", etcd.address, etcd.port)) - .join(","), - ); - } + match config.meta_backend.to_ascii_lowercase().as_str() { + "memory" => { + cmd.arg("--backend").arg("mem"); + } + "etcd" => { + let etcd_config = config.provide_etcd_backend.as_ref().unwrap(); + assert!(!etcd_config.is_empty()); + is_persistent_meta_store = true; + + cmd.arg("--backend") + .arg("etcd") + .arg("--etcd-endpoints") + .arg( + etcd_config + .iter() + .map(|etcd| format!("{}:{}", etcd.address, etcd.port)) + .join(","), + ); + } + "sqlite" => { + let sqlite_config = config.provide_sqlite_backend.as_ref().unwrap(); + assert_eq!(sqlite_config.len(), 1); + is_persistent_meta_store = true; + + let prefix_data = env::var("PREFIX_DATA")?; + let file_path = PathBuf::from(&prefix_data) + .join(&sqlite_config[0].id) + .join(&sqlite_config[0].file); + cmd.arg("--backend") + .arg("sql") + .arg("--sql-endpoint") + .arg(format!("sqlite://{}?mode=rwc", file_path.display())); + } + "postgres" => { + let pg_config = config.provide_postgres_backend.as_ref().unwrap(); + assert_eq!(pg_config.len(), 1); + is_persistent_meta_store = true; + + cmd.arg("--backend") + .arg("sql") + .arg("--sql-endpoint") + .arg(format!( + "postgres://{}:{}@{}:{}/{}", + pg_config[0].user, + pg_config[0].password, + pg_config[0].address, + pg_config[0].port, + pg_config[0].database + )); + } + backend => { + return Err(anyhow!("unsupported meta backend {}", backend)); } } diff --git a/src/risedevtool/src/task/postgres_service.rs b/src/risedevtool/src/task/postgres_service.rs new file mode 100644 index 000000000000..4a6d6571c82c --- /dev/null +++ b/src/risedevtool/src/task/postgres_service.rs @@ -0,0 +1,55 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::task::docker_service::{DockerService, DockerServiceConfig}; +use crate::PostgresConfig; + +impl DockerServiceConfig for PostgresConfig { + fn id(&self) -> String { + self.id.clone() + } + + fn is_user_managed(&self) -> bool { + self.user_managed + } + + fn image(&self) -> String { + self.image.clone() + } + + fn envs(&self) -> Vec<(String, String)> { + vec![ + ("POSTGRES_HOST_AUTH_METHOD".to_owned(), "trust".to_owned()), + ("POSTGRES_USER".to_owned(), self.user.clone()), + ("POSTGRES_PASSWORD".to_owned(), self.password.clone()), + ("POSTGRES_DB".to_owned(), self.database.clone()), + ( + "POSTGRES_INITDB_ARGS".to_owned(), + "--encoding=UTF-8 --lc-collate=C --lc-ctype=C".to_owned(), + ), + ] + } + + fn ports(&self) -> Vec<(String, String)> { + vec![(format!("{}:{}", self.address, self.port), "5432".to_owned())] + } + + fn data_path(&self) -> Option { + self.persist_data + .then(|| "/var/lib/postgresql/data".to_owned()) + } +} + +/// Docker-backed PostgreSQL service. +pub type PostgresService = DockerService;