diff --git a/Cargo.lock b/Cargo.lock index 7ed2fb9dc74..2560897f618 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3893,7 +3893,7 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "greptime-proto" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=a191edaea1089362a86ebc7d8e98ee9a1bd522d1#a191edaea1089362a86ebc7d8e98ee9a1bd522d1" +source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=07de9a47ed46eae0fab4cbc5c32a17c67ed5cb38#07de9a47ed46eae0fab4cbc5c32a17c67ed5cb38" dependencies = [ "prost 0.12.4", "serde", @@ -5280,6 +5280,7 @@ dependencies = [ "common-runtime", "common-telemetry", "common-time", + "common-version", "common-wal", "dashmap", "datatypes", diff --git a/Cargo.toml b/Cargo.toml index 4bf97142290..f4cef06608a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -116,7 +116,7 @@ etcd-client = { git = "https://github.com/MichaelScofield/etcd-client.git", rev fst = "0.4.7" futures = "0.3" futures-util = "0.3" -greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "a191edaea1089362a86ebc7d8e98ee9a1bd522d1" } +greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "07de9a47ed46eae0fab4cbc5c32a17c67ed5cb38" } humantime = "2.1" humantime-serde = "1.1" itertools = "0.10" diff --git a/src/catalog/src/information_schema/cluster_info.rs b/src/catalog/src/information_schema/cluster_info.rs index 2a55810876e..0d80dbe7e4a 100644 --- a/src/catalog/src/information_schema/cluster_info.rs +++ b/src/catalog/src/information_schema/cluster_info.rs @@ -241,8 +241,8 @@ impl InformationSchemaClusterInfoBuilder { return; } - if peer_type == "FRONTEND" { - // Always set peer_id to be -1 for frontends + if peer_type == "FRONTEND" || peer_type == "METASRV" { + // Always set peer_id to be -1 for frontends and metasrvs self.peer_ids.push(Some(-1)); } else { self.peer_ids.push(Some(node_info.peer.id as i64)); diff --git a/src/meta-client/src/client.rs b/src/meta-client/src/client.rs index 685c12ee8a9..788778d49f1 100644 --- a/src/meta-client/src/client.rs +++ b/src/meta-client/src/client.rs @@ -264,29 +264,25 @@ impl ClusterInfo for MetaClient { let mut nodes = if get_metasrv_nodes { let last_activity_ts = -1; // Metasrv does not provide this information. - - // TODO(dennis): Get Metasrv node info - let git_commit = "unknown"; - let version = "unknown"; let start_time_ms = 0; let (leader, followers) = cluster_client.get_metasrv_peers().await?; followers .into_iter() - .map(|peer| NodeInfo { - peer, + .map(|node| NodeInfo { + peer: node.peer.map(|p| p.into()).unwrap_or_default(), last_activity_ts, status: NodeStatus::Metasrv(MetasrvStatus { is_leader: false }), - version: version.to_string(), - git_commit: git_commit.to_string(), + version: node.version, + git_commit: node.git_commit, start_time_ms, }) - .chain(leader.into_iter().map(|leader| NodeInfo { - peer: leader, + .chain(leader.into_iter().map(|node| NodeInfo { + peer: node.peer.map(|p| p.into()).unwrap_or_default(), last_activity_ts, status: NodeStatus::Metasrv(MetasrvStatus { is_leader: true }), - version: version.to_string(), - git_commit: git_commit.to_string(), + version: node.version, + git_commit: node.git_commit, start_time_ms, })) .collect::>() diff --git a/src/meta-client/src/client/cluster.rs b/src/meta-client/src/client/cluster.rs index 56b4d44ce78..60ce52ecb69 100644 --- a/src/meta-client/src/client/cluster.rs +++ b/src/meta-client/src/client/cluster.rs @@ -17,9 +17,8 @@ use std::sync::Arc; use api::greptime_proto::v1; use api::v1::meta::cluster_client::ClusterClient; -use api::v1::meta::{MetasrvPeersRequest, ResponseHeader, Role}; +use api::v1::meta::{MetasrvNodeInfo, MetasrvPeersRequest, ResponseHeader, Role}; use common_grpc::channel_manager::ChannelManager; -use common_meta::peer::Peer; use common_meta::rpc::store::{BatchGetRequest, BatchGetResponse, RangeRequest, RangeResponse}; use common_telemetry::{info, warn}; use snafu::{ensure, ResultExt}; @@ -72,7 +71,9 @@ impl Client { inner.batch_get(req).await } - pub async fn get_metasrv_peers(&self) -> Result<(Option, Vec)> { + pub async fn get_metasrv_peers( + &self, + ) -> Result<(Option, Vec)> { let inner = self.inner.read().await; inner.get_metasrv_peers().await } @@ -225,7 +226,7 @@ impl Inner { .context(ConvertMetaResponseSnafu) } - async fn get_metasrv_peers(&self) -> Result<(Option, Vec)> { + async fn get_metasrv_peers(&self) -> Result<(Option, Vec)> { self.with_retry( "get_metasrv_peers", move |mut client| { @@ -241,10 +242,6 @@ impl Inner { |res| &res.header, ) .await - .map(|res| { - let leader = res.leader.map(|x| x.into()); - let peers = res.followers.into_iter().map(|x| x.into()).collect(); - (leader, peers) - }) + .map(|res| (res.leader, res.followers)) } } diff --git a/src/meta-srv/Cargo.toml b/src/meta-srv/Cargo.toml index 137e45f57a6..28468247de6 100644 --- a/src/meta-srv/Cargo.toml +++ b/src/meta-srv/Cargo.toml @@ -25,6 +25,7 @@ common-procedure.workspace = true common-runtime.workspace = true common-telemetry.workspace = true common-time.workspace = true +common-version.workspace = true common-wal.workspace = true dashmap.workspace = true datatypes.workspace = true diff --git a/src/meta-srv/src/election.rs b/src/meta-srv/src/election.rs index 6562150bfaa..50cac6a27e7 100644 --- a/src/meta-srv/src/election.rs +++ b/src/meta-srv/src/election.rs @@ -21,7 +21,7 @@ use etcd_client::LeaderKey; use tokio::sync::broadcast::Receiver; use crate::error::Result; -use crate::metasrv::LeaderValue; +use crate::metasrv::MetasrvNodeInfo; pub const ELECTION_KEY: &str = "__metasrv_election"; pub const CANDIDATES_ROOT: &str = "__metasrv_election_candidates/"; @@ -71,7 +71,7 @@ pub trait Election: Send + Sync { async fn register_candidate(&self) -> Result<()>; /// Gets all candidates in the election. - async fn all_candidates(&self) -> Result>; + async fn all_candidates(&self) -> Result>; /// Campaign waits to acquire leadership in an election. /// diff --git a/src/meta-srv/src/election/etcd.rs b/src/meta-srv/src/election/etcd.rs index 0b615c78e6e..e15e7cbc276 100644 --- a/src/meta-srv/src/election/etcd.rs +++ b/src/meta-srv/src/election/etcd.rs @@ -27,7 +27,7 @@ use tokio::sync::broadcast::Receiver; use crate::election::{Election, LeaderChangeMessage, CANDIDATES_ROOT, ELECTION_KEY}; use crate::error; use crate::error::Result; -use crate::metasrv::{ElectionRef, LeaderValue}; +use crate::metasrv::{ElectionRef, LeaderValue, MetasrvNodeInfo}; pub struct EtcdElection { leader_value: String, @@ -142,9 +142,19 @@ impl Election for EtcdElection { .context(error::EtcdFailedSnafu)?; let lease_id = res.id(); - // The register info: key is the candidate key, value is its leader value. + // The register info: key is the candidate key, value is its node info(addr, version, git_commit). let key = self.candidate_key().into_bytes(); - let value = self.leader_value.clone().into_bytes(); + let build_info = common_version::build_info(); + let value = MetasrvNodeInfo { + addr: self.leader_value.clone(), + version: build_info.version.to_string(), + git_commit: build_info.commit_short.to_string(), + }; + let value = serde_json::to_string(&value) + .with_context(|_| error::SerializeToJsonSnafu { + input: format!("{value:?}"), + })? + .into_bytes(); // Puts with the lease id self.client .kv_client() @@ -175,7 +185,7 @@ impl Election for EtcdElection { Ok(()) } - async fn all_candidates(&self) -> Result> { + async fn all_candidates(&self) -> Result> { let key = self.candidate_root().into_bytes(); let res = self .client @@ -183,7 +193,19 @@ impl Election for EtcdElection { .get(key, Some(GetOptions::new().with_prefix())) .await .context(error::EtcdFailedSnafu)?; - res.kvs().iter().map(|kv| Ok(kv.value().into())).collect() + + let mut nodes = Vec::with_capacity(res.kvs().len()); + for kv in res.kvs() { + let node = + serde_json::from_slice::(kv.value()).with_context(|_| { + error::DeserializeFromJsonSnafu { + input: String::from_utf8_lossy(kv.value()), + } + })?; + nodes.push(node); + } + + Ok(nodes) } async fn campaign(&self) -> Result<()> { diff --git a/src/meta-srv/src/metasrv.rs b/src/meta-srv/src/metasrv.rs index bb0e0255f1d..311ecd25382 100644 --- a/src/meta-srv/src/metasrv.rs +++ b/src/meta-srv/src/metasrv.rs @@ -207,6 +207,7 @@ impl Context { } } +/// The value of the leader. It is used to store the leader's address. pub struct LeaderValue(pub String); impl> From for LeaderValue { @@ -216,6 +217,29 @@ impl> From for LeaderValue { } } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MetasrvNodeInfo { + // The metasrv's address + pub addr: String, + // The node build version + pub version: String, + // The node build git commit hash + pub git_commit: String, +} + +impl From for api::v1::meta::MetasrvNodeInfo { + fn from(node_info: MetasrvNodeInfo) -> Self { + Self { + peer: Some(api::v1::meta::Peer { + addr: node_info.addr, + ..Default::default() + }), + version: node_info.version, + git_commit: node_info.git_commit, + } + } +} + #[derive(Clone)] pub struct SelectorContext { pub server_addr: String, diff --git a/src/meta-srv/src/service/cluster.rs b/src/meta-srv/src/service/cluster.rs index cf1d25b7341..afc0f608a6b 100644 --- a/src/meta-srv/src/service/cluster.rs +++ b/src/meta-srv/src/service/cluster.rs @@ -14,16 +14,16 @@ use api::v1::meta::{ cluster_server, BatchGetRequest as PbBatchGetRequest, BatchGetResponse as PbBatchGetResponse, - Error, MetasrvPeersRequest, MetasrvPeersResponse, Peer, RangeRequest as PbRangeRequest, - RangeResponse as PbRangeResponse, ResponseHeader, + Error, MetasrvNodeInfo, MetasrvPeersRequest, MetasrvPeersResponse, + RangeRequest as PbRangeRequest, RangeResponse as PbRangeResponse, ResponseHeader, }; use common_telemetry::warn; use snafu::ResultExt; use tonic::{Request, Response}; -use crate::error; use crate::metasrv::Metasrv; use crate::service::GrpcResult; +use crate::{error, metasrv}; #[async_trait::async_trait] impl cluster_server::Cluster for Metasrv { @@ -88,33 +88,30 @@ impl cluster_server::Cluster for Metasrv { return Ok(Response::new(resp)); } + fn make_node_info(addr: String) -> Option { + let build_info = common_version::build_info(); + Some( + metasrv::MetasrvNodeInfo { + addr, + version: build_info.version.to_string(), + git_commit: build_info.commit_short.to_string(), + } + .into(), + ) + } + let (leader, followers) = match self.election() { Some(election) => { let leader = election.leader().await?; - let peers = election.all_candidates().await?; - let followers = peers + let nodes = election.all_candidates().await?; + let followers = nodes .into_iter() - .filter(|peer| peer.0 != leader.0) - .map(|peer| Peer { - addr: peer.0, - ..Default::default() - }) + .filter(|node_info| node_info.addr != leader.0) + .map(api::v1::meta::MetasrvNodeInfo::from) .collect(); - ( - Some(Peer { - addr: leader.0, - ..Default::default() - }), - followers, - ) + (make_node_info(leader.0.clone()), followers) } - None => ( - Some(Peer { - addr: self.options().server_addr.clone(), - ..Default::default() - }), - vec![], - ), + None => (make_node_info(self.options().server_addr.clone()), vec![]), }; let resp = MetasrvPeersResponse { @@ -122,6 +119,7 @@ impl cluster_server::Cluster for Metasrv { leader, followers, }; + Ok(Response::new(resp)) } } diff --git a/tests/cases/distributed/information_schema/cluster_info.result b/tests/cases/distributed/information_schema/cluster_info.result index 3d4158aaffe..6388ebbbf22 100644 --- a/tests/cases/distributed/information_schema/cluster_info.result +++ b/tests/cases/distributed/information_schema/cluster_info.result @@ -18,7 +18,6 @@ DESC TABLE CLUSTER_INFO; +-------------+----------------------+-----+------+---------+---------------+ -- SQLNESS REPLACE version node_version --- SQLNESS REPLACE unknown UNKNOWN -- SQLNESS REPLACE (\s\d\.\d\.\d\s) Version -- SQLNESS REPLACE (\s[a-z0-9]{7}\s) Hash -- SQLNESS REPLACE (\s[\-0-9T:\.]{23}) Start_time @@ -26,10 +25,9 @@ DESC TABLE CLUSTER_INFO; -- SQLNESS REPLACE [\s\-]+ SELECT * FROM CLUSTER_INFO ORDER BY peer_type; -+++++++++|peer_id|peer_type|peer_addr|node_version|git_commit|start_time|uptime|active_time|+++++++++|1|DATANODE|127.0.0.1:4101|Version|Hash|Start_time|Duration|Duration||2|DATANODE|127.0.0.1:4102|Version|Hash|Start_time|Duration|Duration||3|DATANODE|127.0.0.1:4103|Version|Hash|Start_time|Duration|Duration||1|FRONTEND|127.0.0.1:4001|Version|Hash|Start_time|Duration|Duration||0|METASRV|127.0.0.1:3002|UNKNOWN|UNKNOWN||||+++++++++ ++++++++++|peer_id|peer_type|peer_addr|node_version|git_commit|start_time|uptime|active_time|+++++++++|1|DATANODE|127.0.0.1:4101|Version|Hash|Start_time|Duration|Duration||2|DATANODE|127.0.0.1:4102|Version|Hash|Start_time|Duration|Duration||3|DATANODE|127.0.0.1:4103|Version|Hash|Start_time|Duration|Duration||1|FRONTEND|127.0.0.1:4001|Version|Hash|Start_time|Duration|Duration||1|METASRV|127.0.0.1:3002|Version|Hash||||+++++++++ -- SQLNESS REPLACE version node_version --- SQLNESS REPLACE unknown UNKNOWN -- SQLNESS REPLACE (\s\d\.\d\.\d\s) Version -- SQLNESS REPLACE (\s[a-z0-9]{7}\s) Hash -- SQLNESS REPLACE (\s[\-0-9T:\.]{23}) Start_time @@ -37,10 +35,9 @@ SELECT * FROM CLUSTER_INFO ORDER BY peer_type; -- SQLNESS REPLACE [\s\-]+ SELECT * FROM CLUSTER_INFO WHERE PEER_TYPE = 'METASRV' ORDER BY peer_type; -+++++++++|peer_id|peer_type|peer_addr|node_version|git_commit|start_time|uptime|active_time|+++++++++|0|METASRV|127.0.0.1:3002|UNKNOWN|UNKNOWN||||+++++++++ ++++++++++|peer_id|peer_type|peer_addr|node_version|git_commit|start_time|uptime|active_time|+++++++++|1|METASRV|127.0.0.1:3002|Version|Hash||||+++++++++ -- SQLNESS REPLACE version node_version --- SQLNESS REPLACE unknown UNKNOWN -- SQLNESS REPLACE (\s\d\.\d\.\d\s) Version -- SQLNESS REPLACE (\s[a-z0-9]{7}\s) Hash -- SQLNESS REPLACE (\s[\-0-9T:\.]{23}) Start_time @@ -51,7 +48,6 @@ SELECT * FROM CLUSTER_INFO WHERE PEER_TYPE = 'FRONTEND' ORDER BY peer_type; +++++++++|peer_id|peer_type|peer_addr|node_version|git_commit|start_time|uptime|active_time|+++++++++|1|FRONTEND|127.0.0.1:4001|Version|Hash|Start_time|Duration|Duration|+++++++++ -- SQLNESS REPLACE version node_version --- SQLNESS REPLACE unknown UNKNOWN -- SQLNESS REPLACE (\s\d\.\d\.\d\s) Version -- SQLNESS REPLACE (\s[a-z0-9]{7}\s) Hash -- SQLNESS REPLACE (\s[\-0-9T:\.]{23}) Start_time @@ -59,10 +55,9 @@ SELECT * FROM CLUSTER_INFO WHERE PEER_TYPE = 'FRONTEND' ORDER BY peer_type; -- SQLNESS REPLACE [\s\-]+ SELECT * FROM CLUSTER_INFO WHERE PEER_TYPE != 'FRONTEND' ORDER BY peer_type; -+++++++++|peer_id|peer_type|peer_addr|node_version|git_commit|start_time|uptime|active_time|+++++++++|1|DATANODE|127.0.0.1:4101|Version|Hash|Start_time|Duration|Duration||2|DATANODE|127.0.0.1:4102|Version|Hash|Start_time|Duration|Duration||3|DATANODE|127.0.0.1:4103|Version|Hash|Start_time|Duration|Duration||0|METASRV|127.0.0.1:3002|UNKNOWN|UNKNOWN||||+++++++++ ++++++++++|peer_id|peer_type|peer_addr|node_version|git_commit|start_time|uptime|active_time|+++++++++|1|DATANODE|127.0.0.1:4101|Version|Hash|Start_time|Duration|Duration||2|DATANODE|127.0.0.1:4102|Version|Hash|Start_time|Duration|Duration||3|DATANODE|127.0.0.1:4103|Version|Hash|Start_time|Duration|Duration||1|METASRV|127.0.0.1:3002|Version|Hash||||+++++++++ -- SQLNESS REPLACE version node_version --- SQLNESS REPLACE unknown UNKNOWN -- SQLNESS REPLACE (\s\d\.\d\.\d\s) Version -- SQLNESS REPLACE (\s[a-z0-9]{7}\s) Hash -- SQLNESS REPLACE (\s[\-0-9T:\.]{23}) Start_time diff --git a/tests/cases/distributed/information_schema/cluster_info.sql b/tests/cases/distributed/information_schema/cluster_info.sql index 2ac906d05db..c26e4854bcc 100644 --- a/tests/cases/distributed/information_schema/cluster_info.sql +++ b/tests/cases/distributed/information_schema/cluster_info.sql @@ -3,7 +3,6 @@ USE INFORMATION_SCHEMA; DESC TABLE CLUSTER_INFO; -- SQLNESS REPLACE version node_version --- SQLNESS REPLACE unknown UNKNOWN -- SQLNESS REPLACE (\s\d\.\d\.\d\s) Version -- SQLNESS REPLACE (\s[a-z0-9]{7}\s) Hash -- SQLNESS REPLACE (\s[\-0-9T:\.]{23}) Start_time @@ -12,7 +11,6 @@ DESC TABLE CLUSTER_INFO; SELECT * FROM CLUSTER_INFO ORDER BY peer_type; -- SQLNESS REPLACE version node_version --- SQLNESS REPLACE unknown UNKNOWN -- SQLNESS REPLACE (\s\d\.\d\.\d\s) Version -- SQLNESS REPLACE (\s[a-z0-9]{7}\s) Hash -- SQLNESS REPLACE (\s[\-0-9T:\.]{23}) Start_time @@ -21,7 +19,6 @@ SELECT * FROM CLUSTER_INFO ORDER BY peer_type; SELECT * FROM CLUSTER_INFO WHERE PEER_TYPE = 'METASRV' ORDER BY peer_type; -- SQLNESS REPLACE version node_version --- SQLNESS REPLACE unknown UNKNOWN -- SQLNESS REPLACE (\s\d\.\d\.\d\s) Version -- SQLNESS REPLACE (\s[a-z0-9]{7}\s) Hash -- SQLNESS REPLACE (\s[\-0-9T:\.]{23}) Start_time @@ -30,7 +27,6 @@ SELECT * FROM CLUSTER_INFO WHERE PEER_TYPE = 'METASRV' ORDER BY peer_type; SELECT * FROM CLUSTER_INFO WHERE PEER_TYPE = 'FRONTEND' ORDER BY peer_type; -- SQLNESS REPLACE version node_version --- SQLNESS REPLACE unknown UNKNOWN -- SQLNESS REPLACE (\s\d\.\d\.\d\s) Version -- SQLNESS REPLACE (\s[a-z0-9]{7}\s) Hash -- SQLNESS REPLACE (\s[\-0-9T:\.]{23}) Start_time @@ -39,7 +35,6 @@ SELECT * FROM CLUSTER_INFO WHERE PEER_TYPE = 'FRONTEND' ORDER BY peer_type; SELECT * FROM CLUSTER_INFO WHERE PEER_TYPE != 'FRONTEND' ORDER BY peer_type; -- SQLNESS REPLACE version node_version --- SQLNESS REPLACE unknown UNKNOWN -- SQLNESS REPLACE (\s\d\.\d\.\d\s) Version -- SQLNESS REPLACE (\s[a-z0-9]{7}\s) Hash -- SQLNESS REPLACE (\s[\-0-9T:\.]{23}) Start_time