From 99d30c75531a2a22322fa1c7b11bbcfb40b1ae7f Mon Sep 17 00:00:00 2001 From: onewe Date: Thu, 21 Sep 2023 22:33:50 +0800 Subject: [PATCH 1/2] Refactor: refactor Cluster component - add p2c loadbalance component - add simple router component - add replay body component - add failover component - add service directory compoent --- Cargo.toml | 1 + common/base/src/lib.rs | 4 +- common/base/src/param.rs | 12 + common/base/src/svc.rs | 7 + dubbo-build/src/client.rs | 6 - dubbo/Cargo.toml | 3 +- dubbo/src/cluster/clone_body.rs | 364 ++++++++++++++ dubbo/src/cluster/clone_invoker.rs | 248 +++++++++ dubbo/src/cluster/directory.rs | 382 -------------- dubbo/src/cluster/failover.rs | 112 +++++ dubbo/src/cluster/loadbalance/impls/mod.rs | 18 - dubbo/src/cluster/loadbalance/impls/random.rs | 59 --- .../cluster/loadbalance/impls/roundrobin.rs | 85 ---- dubbo/src/cluster/loadbalance/mod.rs | 42 -- dubbo/src/cluster/loadbalance/types.rs | 42 -- dubbo/src/cluster/mod.rs | 475 +++--------------- .../router/condition/condition_router.rs | 60 --- dubbo/src/cluster/router/condition/matcher.rs | 94 ---- dubbo/src/cluster/router/condition/mod.rs | 3 - .../cluster/router/condition/single_router.rs | 225 --------- .../router/manager/condition_manager.rs | 75 --- dubbo/src/cluster/router/manager/mod.rs | 3 - .../cluster/router/manager/router_manager.rs | 161 ------ .../src/cluster/router/manager/tag_manager.rs | 27 - dubbo/src/cluster/router/mod.rs | 25 - .../cluster/router/nacos_config_center/mod.rs | 1 - .../nacos_config_center/nacos_client.rs | 132 ----- dubbo/src/cluster/router/router_chain.rs | 74 --- dubbo/src/cluster/router/tag/mod.rs | 1 - dubbo/src/cluster/router/tag/tag_router.rs | 70 --- dubbo/src/cluster/router/utils.rs | 16 - dubbo/src/codegen.rs | 4 +- dubbo/src/directory/mod.rs | 323 ++++++++++++ dubbo/src/invocation.rs | 4 +- dubbo/src/lib.rs | 3 + dubbo/src/loadbalancer/mod.rs | 103 ++++ dubbo/src/protocol/mod.rs | 9 +- dubbo/src/protocol/triple/triple_invoker.rs | 19 +- dubbo/src/registry/mod.rs | 2 + dubbo/src/registry/n_registry.rs | 75 +++ dubbo/src/route/mod.rs | 135 +++++ dubbo/src/triple/client/builder.rs | 13 +- dubbo/src/triple/client/mod.rs | 2 +- dubbo/src/triple/client/replay.rs | 441 ---------------- dubbo/src/triple/client/triple.rs | 43 +- examples/greeter/src/greeter/client.rs | 15 +- registry/nacos/Cargo.toml | 2 +- 47 files changed, 1490 insertions(+), 2530 deletions(-) create mode 100644 common/base/src/param.rs create mode 100644 common/base/src/svc.rs create mode 100644 dubbo/src/cluster/clone_body.rs create mode 100644 dubbo/src/cluster/clone_invoker.rs delete mode 100644 dubbo/src/cluster/directory.rs create mode 100644 dubbo/src/cluster/failover.rs delete mode 100644 dubbo/src/cluster/loadbalance/impls/mod.rs delete mode 100644 dubbo/src/cluster/loadbalance/impls/random.rs delete mode 100644 dubbo/src/cluster/loadbalance/impls/roundrobin.rs delete mode 100644 dubbo/src/cluster/loadbalance/mod.rs delete mode 100644 dubbo/src/cluster/loadbalance/types.rs delete mode 100644 dubbo/src/cluster/router/condition/condition_router.rs delete mode 100644 dubbo/src/cluster/router/condition/matcher.rs delete mode 100644 dubbo/src/cluster/router/condition/mod.rs delete mode 100644 dubbo/src/cluster/router/condition/single_router.rs delete mode 100644 dubbo/src/cluster/router/manager/condition_manager.rs delete mode 100644 dubbo/src/cluster/router/manager/mod.rs delete mode 100644 dubbo/src/cluster/router/manager/router_manager.rs delete mode 100644 dubbo/src/cluster/router/manager/tag_manager.rs delete mode 100644 dubbo/src/cluster/router/mod.rs delete mode 100644 dubbo/src/cluster/router/nacos_config_center/mod.rs delete mode 100644 dubbo/src/cluster/router/nacos_config_center/nacos_client.rs delete mode 100644 dubbo/src/cluster/router/router_chain.rs delete mode 100644 dubbo/src/cluster/router/tag/mod.rs delete mode 100644 dubbo/src/cluster/router/tag/tag_router.rs delete mode 100644 dubbo/src/cluster/router/utils.rs create mode 100644 dubbo/src/directory/mod.rs create mode 100644 dubbo/src/loadbalancer/mod.rs create mode 100644 dubbo/src/registry/n_registry.rs create mode 100644 dubbo/src/route/mod.rs delete mode 100644 dubbo/src/triple/client/replay.rs diff --git a/Cargo.toml b/Cargo.toml index 5270ba42..0abff00d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,5 @@ [workspace] +resolver = "2" members = [ "common/logger", "common/utils", diff --git a/common/base/src/lib.rs b/common/base/src/lib.rs index b97b342f..8b488501 100644 --- a/common/base/src/lib.rs +++ b/common/base/src/lib.rs @@ -21,6 +21,8 @@ pub mod constants; pub mod node; pub mod url; +pub mod param; +pub mod svc; pub use node::Node; -pub use url::Url; +pub use url::Url; \ No newline at end of file diff --git a/common/base/src/param.rs b/common/base/src/param.rs new file mode 100644 index 00000000..b57f98eb --- /dev/null +++ b/common/base/src/param.rs @@ -0,0 +1,12 @@ +pub trait Param { + + fn param(&self) -> T; +} + + +impl Param for T { + + fn param(&self) -> T::Owned { + self.to_owned() + } +} diff --git a/common/base/src/svc.rs b/common/base/src/svc.rs new file mode 100644 index 00000000..a4b8995a --- /dev/null +++ b/common/base/src/svc.rs @@ -0,0 +1,7 @@ +pub trait NewService { + + type Service; + + fn new_service(&self, target: T) -> Self::Service; + +} \ No newline at end of file diff --git a/dubbo-build/src/client.rs b/dubbo-build/src/client.rs index bfcfe45b..418bc10e 100644 --- a/dubbo-build/src/client.rs +++ b/dubbo-build/src/client.rs @@ -78,12 +78,6 @@ pub fn generate( } } - // pub fn build(builder: ClientBuilder) -> Self { - // Self { - // inner: TripleClient::new(builder), - // } - // } - pub fn new(builder: ClientBuilder) -> Self { Self { inner: TripleClient::new(builder), diff --git a/dubbo/Cargo.toml b/dubbo/Cargo.toml index 43be50b9..402dc171 100644 --- a/dubbo/Cargo.toml +++ b/dubbo/Cargo.toml @@ -14,13 +14,14 @@ hyper = { version = "0.14.26", features = ["full"] } http = "0.2" tower-service.workspace = true http-body = "0.4.4" -tower = { workspace = true, features = ["timeout", "ready-cache","discover"] } +tower = { workspace = true, features = ["timeout", "ready-cache","discover","retry"] } futures-util = "0.3.23" futures-core ="0.3.23" argh = "0.1" rustls-pemfile = "1.0.0" tokio-rustls="0.23.4" tokio = { version = "1.0", features = [ "rt-multi-thread", "time", "fs", "macros", "net", "signal", "full" ] } +tokio-util = "0.7.9" prost = "0.10.4" async-trait = "0.1.56" tower-layer.workspace = true diff --git a/dubbo/src/cluster/clone_body.rs b/dubbo/src/cluster/clone_body.rs new file mode 100644 index 00000000..37db93ac --- /dev/null +++ b/dubbo/src/cluster/clone_body.rs @@ -0,0 +1,364 @@ +use std::{ + collections::VecDeque, + pin::Pin, + sync::{Arc, Mutex}, + task::{Context, Poll}, +}; + +use bytes::{Buf, BufMut, Bytes, BytesMut}; +use futures_core::ready; + +use http::HeaderMap; +use http_body::Body; +use pin_project::pin_project; +use thiserror::Error; + +use crate::StdError; + + +#[derive(Error, Debug)] +#[error("buffered body reach max capacity.")] +pub struct ReachMaxCapacityError; + +pub struct BufferedBody { + shared: Arc>>>, + owned: Option>, + replay_body: bool, + replay_trailers: bool, + is_empty: bool, + size_hint: http_body::SizeHint, +} + +pub struct OwnedBufferedBody { + body: B, + trailers: Option, + buf: InnerBuffer, +} + + + +impl BufferedBody { + + pub fn new(body: B, buf_size: usize) -> Self { + let size_hint = body.size_hint(); + let is_empty = body.is_end_stream(); + BufferedBody { + shared: Default::default(), + owned: Some(OwnedBufferedBody { + body, + trailers: None, + buf: InnerBuffer { + bufs: Default::default(), + capacity: buf_size, + }, + }), + replay_body: false, + replay_trailers: false, + is_empty, + size_hint, + } + } + + pub fn acquire_owned_body(&mut self) -> &mut OwnedBufferedBody { + self.owned.get_or_insert_with(|| { + let lock = self.shared.lock(); + if let Err(e) = lock { + panic!("buffered body get shared data lock failed. {}", e); + } + let mut data = lock.unwrap(); + + data.take().expect("cannot get shared buffered body.") + }) + } +} + +impl Clone for BufferedBody { + + fn clone(&self) -> Self { + Self { + shared: self.shared.clone(), + owned: None, + replay_body: true, + replay_trailers: true, + is_empty: self.is_empty, + size_hint: self.size_hint, + } + } +} + +impl Drop for BufferedBody { + fn drop(&mut self) { + if let Some(owned) = self.owned.take() { + let lock = self.shared.lock(); + if let Ok(mut lock) = lock { + *lock = Some(owned); + } + } + } +} + +impl Body for BufferedBody +where + B: http_body::Body + Unpin, + B::Error: Into, +{ + type Data = BytesData; + type Error = StdError; + + fn poll_data( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll>> { + let mut_self = self.get_mut(); + let owned_body = mut_self.acquire_owned_body(); + + if mut_self.replay_body { + mut_self.replay_body = false; + if owned_body.buf.has_remaining() { + return Poll::Ready(Some(Ok(BytesData::BufferedBytes( + owned_body.buf.clone(), + )))); + } + + if owned_body.buf.is_capped() { + return Poll::Ready(Some(Err(ReachMaxCapacityError.into()))); + } + } + + if mut_self.is_empty { + return Poll::Ready(None); + } + + let mut data = { + let pin = Pin::new(&mut owned_body.body); + let data = ready!(pin.poll_data(cx)); + match data { + Some(Ok(data)) => data, + Some(Err(e)) => return Poll::Ready(Some(Err(e.into()))), + None => { + mut_self.is_empty = true; + return Poll::Ready(None); + } + } + }; + + let len = data.remaining(); + + owned_body.buf.capacity = owned_body.buf.capacity.saturating_sub(len); + + let data = if owned_body.buf.is_capped() { + if owned_body.buf.has_remaining() { + owned_body.buf.bufs = VecDeque::default(); + } + data.copy_to_bytes(len) + } else { + owned_body.buf.push_bytes(data.copy_to_bytes(len)) + }; + + Poll::Ready(Some(Ok(BytesData::OriginBytes(data)))) + + + } + + fn poll_trailers( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, Self::Error>> { + let mut_self = self.get_mut(); + let owned_body = mut_self.acquire_owned_body(); + + if mut_self.replay_trailers { + mut_self.replay_trailers = false; + if let Some(ref trailers) = owned_body.trailers { + return Poll::Ready(Ok(Some(trailers.clone()))); + } + } + + let mut_body = &mut owned_body.body; + if !mut_body.is_end_stream() { + let trailers = ready!(Pin::new(mut_body).poll_trailers(cx)).map(|trailers| { + owned_body.trailers = trailers.clone(); + trailers + }); + return Poll::Ready(trailers.map_err(|e|e.into())); + } + + Poll::Ready(Ok(None)) + } + + fn is_end_stream(&self) -> bool { + if self.is_empty { + return true; + } + + let is_end = self.owned.as_ref() + .map(|owned|owned.body.is_end_stream()) + .unwrap_or(false); + + !self.replay_body && !self.replay_trailers && is_end + } + + fn size_hint(&self) -> http_body::SizeHint { + self.size_hint + } + + +} + + + +#[derive(Clone)] +pub struct InnerBuffer { + bufs: VecDeque, + capacity: usize, +} + +impl InnerBuffer { + pub fn push_bytes(&mut self, bytes: Bytes) -> Bytes { + self.bufs.push_back(bytes.clone()); + bytes + } + + pub fn is_capped(&self) -> bool { + self.capacity == 0 + } +} + +impl Buf for InnerBuffer { + fn remaining(&self) -> usize { + self.bufs.iter().map(|bytes| bytes.remaining()).sum() + } + + fn chunk(&self) -> &[u8] { + self.bufs.front().map(Buf::chunk).unwrap_or(&[]) + } + + fn chunks_vectored<'a>(&'a self, dst: &mut [std::io::IoSlice<'a>]) -> usize { + if dst.is_empty() { + return 0; + } + + let mut filled = 0; + + for bytes in self.bufs.iter() { + filled += bytes.chunks_vectored(&mut dst[filled..]) + } + + filled + } + + fn advance(&mut self, mut cnt: usize) { + while cnt > 0 { + let first = self.bufs.front_mut(); + if first.is_none() { + break; + } + let first = first.unwrap(); + let first_remaining = first.remaining(); + if first_remaining > cnt { + first.advance(cnt); + break; + } + + first.advance(first_remaining); + cnt = cnt - first_remaining; + self.bufs.pop_front(); + } + } + + fn copy_to_bytes(&mut self, len: usize) -> bytes::Bytes { + match self.bufs.front_mut() { + Some(buf) if len <= buf.remaining() => { + let bytes = buf.copy_to_bytes(len); + if buf.remaining() == 0 { + self.bufs.pop_front(); + } + bytes + } + _ => { + let mut bytes = BytesMut::with_capacity(len); + bytes.put(self.take(len)); + bytes.freeze() + } + } + } +} + +pub enum BytesData { + BufferedBytes(InnerBuffer), + OriginBytes(Bytes), +} + +impl Buf for BytesData { + fn remaining(&self) -> usize { + match self { + BytesData::BufferedBytes(bytes) => bytes.remaining(), + BytesData::OriginBytes(bytes) => bytes.remaining(), + } + } + + fn chunk(&self) -> &[u8] { + match self { + BytesData::BufferedBytes(bytes) => bytes.chunk(), + BytesData::OriginBytes(bytes) => bytes.chunk(), + } + } + + fn advance(&mut self, cnt: usize) { + match self { + BytesData::BufferedBytes(bytes) => bytes.advance(cnt), + BytesData::OriginBytes(bytes) => bytes.advance(cnt), + } + } + + fn copy_to_bytes(&mut self, len: usize) -> bytes::Bytes { + match self { + BytesData::BufferedBytes(bytes) => bytes.copy_to_bytes(len), + BytesData::OriginBytes(bytes) => bytes.copy_to_bytes(len), + } + } +} + +#[pin_project] +#[derive(Clone)] +pub struct CloneBody(#[pin] BufferedBody); + +impl CloneBody +where + B: http_body::Body + Unpin, + B::Error: Into, +{ + pub fn new(inner_body: B) -> Self { + let inner_body = BufferedBody::new(inner_body, 1024 * 64); + CloneBody(inner_body) + } +} + +impl Body for CloneBody +where + B: http_body::Body + Unpin, + B::Error: Into, +{ + + type Data = BytesData; + + type Error = StdError; + + fn poll_data( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll>> { + self.project().0.poll_data(cx) + } + + fn poll_trailers( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, Self::Error>> { + self.project().0.poll_trailers(cx) + } + + fn size_hint(&self) -> http_body::SizeHint { + self.0.size_hint() + } +} diff --git a/dubbo/src/cluster/clone_invoker.rs b/dubbo/src/cluster/clone_invoker.rs new file mode 100644 index 00000000..20e18118 --- /dev/null +++ b/dubbo/src/cluster/clone_invoker.rs @@ -0,0 +1,248 @@ +use std::{task::Poll, pin::Pin, mem}; + +use dubbo_logger::tracing::debug; +use futures_core::{Future, TryFuture, ready, future::BoxFuture}; +use futures_util::FutureExt; +use pin_project::pin_project; +use thiserror::Error; +use tokio::{task::JoinHandle, sync::{watch::{Sender, Receiver}, self}}; +use tokio_util::sync::ReusableBoxFuture; +use tower::{ServiceExt, buffer::Buffer}; +use tower_service::Service; + +use crate::StdError; + +enum Inner { + Invalid, + Ready(S), + Pending(JoinHandle>), +} + +#[derive(Debug, Error)] +#[error("the inner service has not got ready yet!")] +struct InnerServiceNotReadyErr; + + + +#[pin_project(project = InnerServiceCallingResponseProj)] +enum InnerServiceCallingResponse { + Call(#[pin] Fut), + Fail +} + +impl Future for InnerServiceCallingResponse +where + Fut: TryFuture, + Fut::Error: Into +{ + type Output = Result; + + fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { + match self.project() { + InnerServiceCallingResponseProj::Call(call) => call.try_poll(cx).map_err(Into::into), + InnerServiceCallingResponseProj::Fail => Poll::Ready(Err(InnerServiceNotReadyErr.into())) + } + } +} + +#[derive(Clone)] +enum ObserveState { + Ready, + Pending, +} + + +struct ReadyService { + inner: Inner, + tx: Sender +} + + +impl ReadyService { + + fn new(inner: S) -> (Self, Receiver) { + let (tx, rx) = sync::watch::channel(ObserveState::Ready); + let ready_service = Self { inner: Inner::Ready(inner), tx}; + (ready_service, rx) + } + +} + +impl Service for ReadyService +where + S: Service + Send + 'static, + >::Error: Into, +{ + type Response = S::Response; + + type Error = StdError; + + type Future = InnerServiceCallingResponse; + + fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { + loop { + match mem::replace(&mut self.inner, Inner::Invalid) { + Inner::Ready(mut svc) => { + let poll_ready = svc.poll_ready(cx); + match poll_ready { + Poll::Pending => { + self.inner = Inner::Pending(tokio::spawn(async move { + let poll_ready = svc.ready().await; + match poll_ready { + Ok(_) => Ok(svc), + Err(err) => { + Err((svc, err.into())) + } + } + })); + + let _ = self.tx.send(ObserveState::Pending); + continue; + } + Poll::Ready(ret) => { + self.inner = Inner::Ready(svc); + + let _ = self.tx.send(ObserveState::Ready); + return Poll::Ready(ret.map_err(Into::into)); + } + } + }, + Inner::Pending(mut join_handle) => { + if let Poll::Ready(res) = join_handle.poll_unpin(cx) { + let (svc, res) = match res { + Err(join_err) => panic!("ReadyService panicked: {join_err}"), + Ok(Err((svc, err))) => (svc, Poll::Ready(Err(err))), + Ok(Ok(svc)) => (svc, Poll::Ready(Ok(()))) + }; + + self.inner = Inner::Ready(svc); + + let _ = self.tx.send(ObserveState::Ready); + return res; + } else { + self.inner = Inner::Pending(join_handle); + + let _ = self.tx.send(ObserveState::Pending); + return Poll::Pending; + } + + }, + Inner::Invalid => panic!("ReadyService panicked: inner state is invalid") + } + } + } + + fn call(&mut self, req: Req) -> Self::Future { + match self.inner { + Inner::Ready(ref mut svc) => InnerServiceCallingResponse::Call(svc.call(req)), + _ => InnerServiceCallingResponse::Fail + } + } +} + + +impl Drop for ReadyService { + fn drop(&mut self) { + if let Inner::Pending(ref handler) = self.inner { + handler.abort(); + } + } +} + +pub struct CloneInvoker +where + Inv: Service + Send + 'static, + Inv::Error: Into + Send + Sync + 'static, + Inv::Future: Send, + Req: Send +{ + inner: Buffer, Req>, + rx: Receiver, + poll: ReusableBoxFuture<'static, ObserveState>, + polling: bool, +} + +impl CloneInvoker +where + Inv: Service + Send + 'static, + Inv::Error: Into + Send + Sync + 'static, + Inv::Future: Send, + Req: Send + 'static +{ + + pub fn new(invoker: Inv) -> Self { + + let (ready_service, rx) = ReadyService::new(invoker); + + let buffer: Buffer, Req> = Buffer::new(ready_service, 1024); + + Self { inner: buffer, rx, polling: false, poll: ReusableBoxFuture::new(futures::future::pending()) } + } +} + +impl Service for CloneInvoker +where + Inv: Service + Send + 'static, + Inv::Error: Into + Send + Sync + 'static, + Inv::Future: Send, + Req: Send + 'static +{ + type Response = Inv::Response; + + type Error = StdError; + + type Future = BoxFuture<'static, Result>; + + fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { + loop { + if !self.polling { + match self.rx.borrow().clone() { + ObserveState::Ready => return self.inner.poll_ready(cx), + ObserveState::Pending => { + self.polling = true; + let mut rx = self.rx.clone(); + self.poll.set(async move { + loop { + let current_state = rx.borrow_and_update().clone(); + if matches!(current_state, ObserveState::Ready) { + return current_state; + } + if let Err(_) = rx.changed().await { + debug!("the readyService has already shutdown!"); + futures::future::pending::().await; + } + } + }); + } + } + + } + + let state = ready!(self.poll.poll_unpin(cx)); + self.polling = false; + + if matches!(state, ObserveState::Pending) { + continue; + } + + return self.inner.poll_ready(cx); + } + } + + fn call(&mut self, req: Req) -> Self::Future { + Box::pin(self.inner.call(req)) + } +} + + +impl Clone for CloneInvoker +where + Inv: Service + Send + 'static, + Inv::Error: Into + Send + Sync + 'static, + Inv::Future: Send, + Req: Send +{ + fn clone(&self) -> Self { + Self { inner: self.inner.clone(), rx: self.rx.clone(), polling: false, poll: ReusableBoxFuture::new(futures::future::pending())} + } +} \ No newline at end of file diff --git a/dubbo/src/cluster/directory.rs b/dubbo/src/cluster/directory.rs deleted file mode 100644 index 547878bf..00000000 --- a/dubbo/src/cluster/directory.rs +++ /dev/null @@ -1,382 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -use std::{ - collections::HashMap, - fmt::Debug, - pin::Pin, - str::FromStr, - sync::{Arc, RwLock}, - task::{Context, Poll}, -}; - -use crate::{ - codegen::TripleInvoker, - protocol::BoxInvoker, - registry::{memory_registry::MemoryNotifyListener, BoxRegistry}, - triple::client::replay::ClonedBody, - StdError, -}; -use dubbo_base::Url; -use dubbo_logger::tracing; -use futures_core::ready; -use tower::{ - discover::{Change, Discover}, - ready_cache::ReadyCache, -}; - -use crate::cluster::Directory; - -/// Directory. -/// -/// [Directory Service](http://en.wikipedia.org/wiki/Directory_service) - -#[derive(Debug, Clone)] -pub struct StaticDirectory { - uri: http::Uri, -} - -impl StaticDirectory { - pub fn new(host: &str) -> StaticDirectory { - let uri = match http::Uri::from_str(host) { - Ok(v) => v, - Err(err) => { - tracing::error!("http uri parse error: {}, host: {}", err, host); - panic!("http uri parse error: {}, host: {}", err, host) - } - }; - StaticDirectory { uri } - } - - pub fn from_uri(uri: &http::Uri) -> StaticDirectory { - StaticDirectory { uri: uri.clone() } - } -} - -impl Directory for StaticDirectory { - fn list(&self, service_name: String) -> Vec { - let url = Url::from_url(&format!( - "tri://{}:{}/{}", - self.uri.host().unwrap(), - self.uri.port().unwrap(), - service_name, - )) - .unwrap(); - let invoker = Box::new(TripleInvoker::new(url)); - vec![invoker] - } -} - -#[derive(Debug, Clone)] -pub struct RegistryDirectory { - registry: Arc, - service_instances: Arc>>>, -} - -impl RegistryDirectory { - pub fn new(registry: BoxRegistry) -> RegistryDirectory { - RegistryDirectory { - registry: Arc::new(registry), - service_instances: Arc::new(RwLock::new(HashMap::new())), - } - } -} - -impl Directory for RegistryDirectory { - fn list(&self, service_name: String) -> Vec { - let url = Url::from_url(&format!( - "triple://{}:{}/{}", - "127.0.0.1", "8888", service_name - )) - .unwrap(); - - self.registry - .subscribe( - url, - Arc::new(MemoryNotifyListener { - service_instances: Arc::clone(&self.service_instances), - }), - ) - .expect("subscribe"); - - let map = self - .service_instances - .read() - .expect("service_instances.read"); - let binding = Vec::new(); - let url_vec = map.get(&service_name).unwrap_or(&binding); - // url_vec.to_vec() - let mut invokers: Vec = vec![]; - for item in url_vec.iter() { - invokers.push(Box::new(TripleInvoker::new(item.clone()))); - } - - invokers - } -} - -pub struct ServiceNameDirectory { - cache: ReadyCache>, - discover: D, -} - -impl ServiceNameDirectory -where - D: Discover + Unpin, - D::Error: Into, -{ - pub fn new(discover: D) -> Self { - Self { - cache: ReadyCache::default(), - discover, - } - } - - fn update_cache(&mut self, cx: &mut Context<'_>) -> Poll> { - loop { - let discover = Pin::new(&mut self.discover); - let update = ready!(discover.poll_discover(cx)); - match update { - Some(update) => { - let change = match update { - Err(_) => continue, - Ok(change) => change, - }; - - match change { - Change::Insert(key, invoker) => { - self.cache.push(key, invoker); - } - Change::Remove(key) => { - self.cache.evict(&key); - } - } - } - None => break, - } - } - - // poll pending - let _ = ready!(self.cache.poll_pending(cx))?; - - Poll::Ready(Ok(())) - } - - pub fn list(&mut self, cx: &mut Context<'_>) -> Poll, StdError>> { - let _ = self.update_cache(cx)?; - - let ready_len = self.cache.ready_len(); - - if ready_len == 0 { - return Poll::Pending; - } - - let mut invoker_list = Vec::with_capacity(ready_len); - - for idx in 0..ready_len { - let check = self.cache.check_ready_index(cx, idx); - let is_ready = match check { - Ok(is_ready) => is_ready, - Err(_) => false, - }; - if !is_ready { - continue; - } - - let invoker_url = match self.cache.get_ready_index(idx) { - None => continue, - Some((k, _)) => k.clone(), - }; - - invoker_list.push(invoker_url); - } - - Poll::Ready(Ok(invoker_list)) - } -} - -#[cfg(test)] -pub mod tests { - use std::{ - convert::Infallible, - pin::Pin, - task::{Context, Poll}, - }; - - use crate::{boxed, cluster::directory::ServiceNameDirectory}; - use bytes::Buf; - use futures_core::Stream; - use futures_util::future::poll_fn; - use http::StatusCode; - use http_body::Body; - use tower::{discover::Change, Service}; - - use dubbo_base::Url; - - use crate::{codegen::Invoker, protocol::BoxInvoker, triple::client::replay::ClonedBody}; - - pub struct MockStaticServiceList - where - T: IntoIterator, - { - iter: T::IntoIter, - } - - impl MockStaticServiceList - where - T: IntoIterator, - { - pub fn new(list: T) -> Self { - let iter = list.into_iter(); - - Self { iter } - } - } - - impl Stream for MockStaticServiceList - where - T: IntoIterator, - T::IntoIter: Unpin, - { - type Item = Result, Infallible>; - - fn poll_next(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll> { - let mut_self_ref = self.get_mut(); - let mut_iter_ref = &mut mut_self_ref.iter; - - match mut_iter_ref.next() { - Some(next) => { - let invoker_url = next.get_url(); - let raw_url_string = invoker_url.raw_url_string(); - return Poll::Ready(Some(Ok(Change::Insert(raw_url_string, next)))); - } - None => Poll::Ready(None), - } - } - } - - #[derive(Debug)] - struct MockInvoker(u8, bool); - - impl Invoker> for MockInvoker { - fn get_url(&self) -> dubbo_base::Url { - let str = format!( - "triple://127.0.0.1:8888/failover_cluster_service/directory/{}", - self.0 - ); - Url::from_url(str.as_str()).unwrap() - } - } - - impl Service> for MockInvoker { - type Response = http::Response; - - type Error = crate::Error; - - type Future = crate::BoxFuture, crate::Error>; - - fn poll_ready(&mut self, _: &mut Context<'_>) -> Poll> { - if !self.1 { - return Poll::Pending; - } - - Poll::Ready(Ok(())) - } - - fn call(&mut self, req: http::Request) -> Self::Future { - Box::pin(async move { - let mut body = req.into_body(); - - let mut body_data = poll_fn(|cx| Pin::new(&mut body).poll_data(cx)) - .await - .unwrap() - .unwrap(); - - let len = body_data.remaining(); - let bytes = body_data.copy_to_bytes(len); - - let req_str = String::from_utf8(bytes.to_vec()).unwrap(); - println!("req: {}", req_str); - let echo_data = format!("echo: {}", req_str); - let response = http::Response::builder() - .status(StatusCode::OK) - .body(boxed(echo_data)) - .unwrap(); - - return Ok(response); - }) - } - } - - #[tokio::test] - async fn test_directory() { - let invoker_list = vec![ - Box::new(MockInvoker(1, true)) as BoxInvoker, - Box::new(MockInvoker(2, true)) as BoxInvoker, - Box::new(MockInvoker(3, true)) as BoxInvoker, - Box::new(MockInvoker(4, true)) as BoxInvoker, - Box::new(MockInvoker(5, true)) as BoxInvoker, - Box::new(MockInvoker(6, true)) as BoxInvoker, - ]; - let service_list = MockStaticServiceList::new(invoker_list); - - let mut service_name_directory = ServiceNameDirectory::new(service_list); - - let list = poll_fn(|cx| service_name_directory.list(cx)).await; - - assert!(list.is_ok()); - - let list = list.unwrap(); - - assert!(!list.is_empty()); - - assert_eq!(6, list.len()); - - for uri in list { - println!("invoker uri: {}", uri); - } - } - - #[tokio::test] - async fn test_ready_in_any_order() { - let invoker_list = vec![ - Box::new(MockInvoker(1, true)) as BoxInvoker, - Box::new(MockInvoker(2, false)) as BoxInvoker, - Box::new(MockInvoker(3, false)) as BoxInvoker, - Box::new(MockInvoker(4, true)) as BoxInvoker, - Box::new(MockInvoker(5, false)) as BoxInvoker, - Box::new(MockInvoker(6, true)) as BoxInvoker, - ]; - let service_list = MockStaticServiceList::new(invoker_list); - - let mut service_name_directory = ServiceNameDirectory::new(service_list); - - let list = poll_fn(|cx| service_name_directory.list(cx)).await; - - assert!(list.is_ok()); - - let list = list.unwrap(); - - assert!(!list.is_empty()); - - assert_eq!(3, list.len()); - - for uri in list { - println!("invoker uri: {}", uri); - } - } -} diff --git a/dubbo/src/cluster/failover.rs b/dubbo/src/cluster/failover.rs new file mode 100644 index 00000000..033c2b09 --- /dev/null +++ b/dubbo/src/cluster/failover.rs @@ -0,0 +1,112 @@ +use std::{task::Poll, marker::PhantomData}; + +use dubbo_base::{svc::NewService, param::Param}; +use futures_util::future; +use http::Request; +use tower::{ServiceExt, retry::Retry, util::Oneshot}; +use tower_service::Service; + +use crate::{codegen::RpcInvocation, StdError}; + +pub struct Failover { + inner: N // loadbalancer service +} + +#[derive(Clone)] +pub struct FailoverPolicy; + + +pub struct NewFailover { + inner: N, // new loadbalancer service + _mark: PhantomData +} + + +impl NewFailover { + + pub fn new(inner: N) -> Self { + NewFailover { + inner, + _mark: PhantomData, + } + } +} + +impl NewService for NewFailover +where + T: Param, + // new loadbalancer service + N: NewService, + // loadbalancer service + N::Service: Service + Send + Clone + 'static, + >::Future: Send, + >::Error: Into + Send + Sync +{ + + type Service = Failover; + + fn new_service(&self, target: T) -> Self::Service { + // loadbalancer service + let inner = self.inner.new_service(target); + + Failover { + inner + } + } + +} + + +impl tower::retry::Policy, Res, E> for FailoverPolicy +where + B: http_body::Body + Clone, +{ + + type Future = future::Ready; + + fn retry(&self, req: &Request, result: Result<&Res, &E>) -> Option { + //TODO some error handling or logging + match result { + Ok(_) => None, + Err(_) => Some(future::ready(self.clone())) + } + } + + fn clone_request(&self, req: &Request) -> Option> { + let mut clone = http::Request::new(req.body().clone()); + *clone.method_mut() = req.method().clone(); + *clone.uri_mut() = req.uri().clone(); + *clone.headers_mut() = req.headers().clone(); + *clone.version_mut() = req.version(); + + + Some(clone) + } +} + + + +impl Service> for Failover +where + // B is CloneBody + B: http_body::Body + Clone, + // loadbalancer service + N: Service> + Clone + 'static , + N::Error: Into, + N::Future: Send +{ + type Response = N::Response; + + type Error = N::Error; + + type Future = Oneshot, Request>; + + fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, req: Request) -> Self::Future { + let retry = Retry::new(FailoverPolicy, self.inner.clone()); + retry.oneshot(req) + } +} \ No newline at end of file diff --git a/dubbo/src/cluster/loadbalance/impls/mod.rs b/dubbo/src/cluster/loadbalance/impls/mod.rs deleted file mode 100644 index 5a84af8c..00000000 --- a/dubbo/src/cluster/loadbalance/impls/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -pub mod random; -pub mod roundrobin; diff --git a/dubbo/src/cluster/loadbalance/impls/random.rs b/dubbo/src/cluster/loadbalance/impls/random.rs deleted file mode 100644 index 3e1cf651..00000000 --- a/dubbo/src/cluster/loadbalance/impls/random.rs +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -use dubbo_base::Url; -use std::{ - fmt::{Debug, Formatter}, - sync::Arc, -}; - -use crate::{ - cluster::loadbalance::types::{LoadBalance, Metadata}, - codegen::RpcInvocation, -}; - -pub struct RandomLoadBalance { - pub metadata: Metadata, -} - -impl Default for RandomLoadBalance { - fn default() -> Self { - RandomLoadBalance { - metadata: Metadata::new("random"), - } - } -} - -impl Debug for RandomLoadBalance { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "RandomLoadBalance") - } -} - -impl LoadBalance for RandomLoadBalance { - fn select( - &self, - invokers: Arc>, - _url: Option, - _invocation: Arc, - ) -> Option { - if invokers.is_empty() { - return None; - } - let index = rand::random::() % invokers.len(); - Some(invokers[index].clone()) - } -} diff --git a/dubbo/src/cluster/loadbalance/impls/roundrobin.rs b/dubbo/src/cluster/loadbalance/impls/roundrobin.rs deleted file mode 100644 index 5fd0ed49..00000000 --- a/dubbo/src/cluster/loadbalance/impls/roundrobin.rs +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -use dubbo_base::Url; -use std::{ - collections::HashMap, - fmt::{Debug, Formatter}, - sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, RwLock, - }, -}; - -use crate::{ - cluster::loadbalance::types::{LoadBalance, Metadata}, - codegen::RpcInvocation, -}; - -pub struct RoundRobinLoadBalance { - pub metadata: Metadata, - pub counter_map: RwLock>, -} - -impl Default for RoundRobinLoadBalance { - fn default() -> Self { - RoundRobinLoadBalance { - metadata: Metadata::new("roundrobin"), - counter_map: RwLock::new(HashMap::new()), - } - } -} - -impl Debug for RoundRobinLoadBalance { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "RoundRobinLoadBalance") - } -} - -impl RoundRobinLoadBalance { - fn guarantee_counter_key(&self, key: &str) { - let contained = self.counter_map.try_read().unwrap().contains_key(key); - if !contained { - self.counter_map - .try_write() - .unwrap() - .insert(key.to_string(), AtomicUsize::new(0)); - } - } -} - -impl LoadBalance for RoundRobinLoadBalance { - fn select( - &self, - invokers: Arc>, - _url: Option, - invocation: Arc, - ) -> Option { - if invokers.is_empty() { - return None; - } - let fingerprint = invocation.unique_fingerprint(); - self.guarantee_counter_key(fingerprint.as_str()); - let index = self - .counter_map - .try_read() - .unwrap() - .get(fingerprint.as_str())? - .fetch_add(1, Ordering::SeqCst) - % invokers.len(); - Some(invokers[index].clone()) - } -} diff --git a/dubbo/src/cluster/loadbalance/mod.rs b/dubbo/src/cluster/loadbalance/mod.rs deleted file mode 100644 index 1d04f7fc..00000000 --- a/dubbo/src/cluster/loadbalance/mod.rs +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -use std::collections::HashMap; - -use lazy_static::lazy_static; - -use crate::cluster::loadbalance::{ - impls::{random::RandomLoadBalance, roundrobin::RoundRobinLoadBalance}, - types::BoxLoadBalance, -}; - -pub mod impls; -pub mod types; - -lazy_static! { - pub static ref LOAD_BALANCE_EXTENSIONS: HashMap = - init_loadbalance_extensions(); -} - -fn init_loadbalance_extensions() -> HashMap { - let mut loadbalance_map: HashMap = HashMap::new(); - loadbalance_map.insert("random".to_string(), Box::new(RandomLoadBalance::default())); - loadbalance_map.insert( - "roundrobin".to_string(), - Box::new(RoundRobinLoadBalance::default()), - ); - loadbalance_map -} diff --git a/dubbo/src/cluster/loadbalance/types.rs b/dubbo/src/cluster/loadbalance/types.rs deleted file mode 100644 index 9273d07e..00000000 --- a/dubbo/src/cluster/loadbalance/types.rs +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -use dubbo_base::Url; -use std::{fmt::Debug, sync::Arc}; - -use crate::codegen::RpcInvocation; - -pub type BoxLoadBalance = Box; - -pub trait LoadBalance: Debug { - fn select( - &self, - invokers: Arc>, - url: Option, - invocation: Arc, - ) -> Option; -} - -pub struct Metadata { - pub name: &'static str, -} - -impl Metadata { - pub fn new(name: &'static str) -> Self { - Metadata { name } - } -} diff --git a/dubbo/src/cluster/mod.rs b/dubbo/src/cluster/mod.rs index 7411a675..c5e50cfa 100644 --- a/dubbo/src/cluster/mod.rs +++ b/dubbo/src/cluster/mod.rs @@ -15,447 +15,88 @@ * limitations under the License. */ -use std::{fmt::Debug, sync::Arc}; +use std::marker::PhantomData; -use dubbo_base::Url; -use thiserror::Error; -use tower::{ready_cache::ReadyCache, ServiceExt}; +use dubbo_base::{svc::NewService, param::Param}; +use http::Request; use tower_service::Service; -use crate::{ - codegen::RpcInvocation, - invocation::Invocation, - protocol::{triple::triple_invoker::TripleInvoker, BoxInvoker, Invoker}, - triple::client::replay::ClonedBody, - StdError, -}; +use crate::{codegen::RpcInvocation, cluster::failover::NewFailover, StdError}; -pub mod directory; -pub mod loadbalance; -pub mod router; +use self::{failover::Failover, clone_body::CloneBody}; + +mod clone_body; +mod clone_invoker; +mod failover; -pub trait Directory: Debug { - fn list(&self, service_name: String) -> Vec; +pub struct NewClusterService { + inner: S, // new loadbalancer service + _mark: PhantomData } -type BoxDirectory = Box; - -pub trait Cluster { - fn join(&self, dir: BoxDirectory) -> BoxInvoker; -} - -#[derive(Debug, Default)] -pub struct MockCluster {} - -impl Cluster for MockCluster { - fn join(&self, dir: BoxDirectory) -> BoxInvoker { - Box::new(FailoverCluster::new(dir)) - } -} - -// 在Cluster上进行缓存Service -#[derive(Debug)] -pub struct FailoverCluster { - dir: Arc, - caches: ReadyCache>, +pub struct ClusterService { + inner: S // failover service } -impl FailoverCluster { - pub fn new(dir: BoxDirectory) -> FailoverCluster { - Self { - dir: Arc::new(dir), - caches: ReadyCache::default(), - } - } -} - -#[derive(Error, Debug)] -#[error("no available service for {0}")] -pub struct NoAvailableServiceErr(String); - -#[derive(Error, Debug)] -#[error("invalid service name {0}")] -pub struct InvalidServiceNameErr(String); - -impl FailoverCluster { - async fn invoke( - req: http::Request, - mut invoker: BoxInvoker, - ) -> Result, (StdError, http::Request)> { - let clone_request = FailoverCluster::clone_request(&req); - let invoker = invoker - .ready() - .await - .map_err(|e| (e, FailoverCluster::clone_request(&req)))?; - let ret = invoker.call(req).await.map_err(|e| (e, clone_request))?; - - Ok(ret) - } - - fn clone_request(req: &http::Request) -> http::Request { - let mut clone = http::Request::new(req.body().clone()); - *clone.method_mut() = req.method().clone(); - *clone.uri_mut() = req.uri().clone(); - *clone.headers_mut() = req.headers().clone(); - *clone.version_mut() = req.version(); - - if let Some(inv) = req.extensions().get::().cloned() { - clone.extensions_mut().insert(inv); - } - - clone - } -} - -impl Service> for FailoverCluster { - type Response = http::Response; - - type Error = crate::Error; - - type Future = crate::BoxFuture; - - fn poll_ready( - &mut self, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - self.caches.poll_pending(cx).map_err(|e| e.into()) - } - - fn call(&mut self, req: http::Request) -> Self::Future { - let inv = req.extensions().get::(); - if inv.is_none() { - return Box::pin(async move { - return Err( - InvalidServiceNameErr("service name must not be null".to_owned()).into(), - ); - }); - } - - let inv = inv.unwrap(); - let service_name = inv.get_target_service_unique_name(); - - let invokers = self.dir.list(service_name.clone()); - - Box::pin(async move { - let mut current_req = req; - let mut last_err = None; - - let is_empty = invokers.is_empty(); - if is_empty { - return Err(NoAvailableServiceErr(service_name).into()); - } - - for invoker in invokers { - match FailoverCluster::invoke(current_req, invoker).await { - Ok(resp) => return Ok(resp), - Err((e, cloned_request)) => { - current_req = cloned_request; - last_err = Some(e); - } - } - } - if last_err.is_none() { - return Err(NoAvailableServiceErr(service_name).into()); +impl NewClusterService +{ + + pub fn layer() -> impl tower_layer::Layer { + tower_layer::layer_fn(|inner: S| { + NewClusterService { + inner, // new loadbalancer service + _mark: PhantomData } - - return Err(last_err.unwrap()); }) } -} - -impl Invoker> for FailoverCluster { - fn get_url(&self) -> dubbo_base::Url { - Url::from_url("triple://127.0.0.1:8888/helloworld.Greeter").unwrap() - } -} - -#[derive(Debug, Default)] -pub struct MockDirectory { - // router_chain: RouterChain, -} - -impl MockDirectory { - pub fn new() -> MockDirectory { - // let router_chain = get_global_router_manager().read().unwrap().get_router_chain(invocation); - Self { - // router_chain - } - } -} - -impl Directory for MockDirectory { - fn list(&self, service_name: String) -> Vec { - // tracing::info!("MockDirectory: {}", meta); - let u = Url::from_url("triple://127.0.0.1:8888/helloworld.Greeter").unwrap(); - vec![Box::new(TripleInvoker::new(u))] - // self.router_chain.route(u, invo); - } -} - - -// #[derive(Debug, Default)] -// pub struct RouterChain { -// router: HashMap, -// invokers: Arc>, -// } - -// impl RouterChain { -// pub fn route(&mut self, url: Url, invo: Arc) -> Arc> { -// let r = self.router.get("mock").unwrap(); -// r.route(self.invokers.clone(), url, invo) -// } -// } - -// pub trait Router: Debug { -// fn route( -// &self, -// invokers: Arc>, -// url: Url, -// invo: Arc, -// ) -> Arc>; -// } - -// pub type BoxRouter = Box; -#[cfg(test)] -pub mod tests { - use std::task::Poll; +} - use bytes::{Buf, BufMut, BytesMut}; - use dubbo_base::Url; - use futures_util::future::poll_fn; - use http::StatusCode; - use http_body::Body; - use thiserror::Error; - use tower::ServiceExt; - use tower_service::Service; +impl NewService for NewClusterService +where + T: Param, + // new loadbalancer service + S: NewService + Clone, + // loadbalancer service + Svc: Service + Clone + Send, + Svc::Error: Into + Send + Sync, + Svc::Future: Send +{ - use crate::{ - boxed, - cluster::FailoverCluster, - codegen::{Invoker, RpcInvocation}, - empty_body, - invocation::Invocation, - triple::client::replay::ClonedBody, - }; + type Service = ClusterService>; - use super::Directory; + fn new_service(&self, target: T) -> Self::Service { + // Failover + let inner = NewFailover::new(self.inner.clone()).new_service(target); - #[derive(Error, Debug)] - #[error("{0}")] - struct NoResponseErr(String); - - #[derive(Debug)] - struct MockDirectory; - - impl Directory for MockDirectory { - fn list(&self, service_name: String) -> Vec { - println!("get invoker list for {}", service_name); - - vec![ - Box::new(MockInvoker(1)), - Box::new(MockInvoker(2)), - Box::new(MockInvoker(3)), - Box::new(MockInvoker(4)), - Box::new(MockInvoker(5)), - ] - } - } - - #[derive(Debug)] - struct MockInvoker(u8); - - impl Invoker> for MockInvoker { - fn get_url(&self) -> dubbo_base::Url { - let str = format!( - "triple://127.0.0.1:8888/failover_cluster_service/{}", - self.0 - ); - Url::from_url(str.as_str()).unwrap() + ClusterService { + inner } } +} + +impl Service> for ClusterService +where + S: Service>>, + B: http_body::Body + Unpin, + B::Error: Into, +{ - impl Service> for MockInvoker { - type Response = http::Response; - - type Error = crate::Error; - - type Future = crate::BoxFuture, crate::Error>; - - fn poll_ready( - &mut self, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - Poll::Ready(Ok(())) - } - - fn call(&mut self, req: http::Request) -> Self::Future { - let inv = req.extensions().get::(); - if inv.is_none() { - return Box::pin(async move { - let response = http::Response::builder() - .status(StatusCode::OK) - .body(empty_body()) - .unwrap(); - - return Ok(response); - }); - } - - let inv = inv.unwrap(); - let method_name = inv.get_method_name(); - if method_name.eq("invoker_request") { - return Box::pin(async move { - let response = http::Response::builder() - .status(StatusCode::OK) - .body(boxed("invoker response".to_owned())) - .unwrap(); - - return Ok(response); - }); - } - - let self_index = self.0; - if method_name.eq("failover_request") { - return Box::pin(async move { - let body = req.into_body(); - let mut pin_body = Box::pin(body); - let ret = poll_fn(|cx| pin_body.as_mut().poll_data(cx)).await; - - if ret.is_none() { - #[derive(Error, Debug)] - #[error("{0}")] - struct BodyIsNoneErr(&'static str); - - return Err(BodyIsNoneErr("body must not be null").into()); - } - - let ret = ret.unwrap(); - - if ret.is_err() { - #[derive(Error, Debug)] - #[error("{0}")] - struct BodyIsErr(&'static str); - return Err(BodyIsErr("body must be ok").into()); - } - - let mut ret = ret.unwrap(); - - let index = ret.get_u8(); - - if index == self_index { - let ret_msg = format!("failover cluster index: {} was invoked", self_index); - let response = http::Response::builder() - .status(StatusCode::OK) - .body(boxed(ret_msg)) - .unwrap(); - - return Ok(response); - } - - #[derive(Error, Debug)] - #[error("{0}")] - struct NotTargetInvoker(String); - - let ret_msg = format!( - "failover cluster index: {} was invoked, but is not target invoker {}", - self_index, index - ); - - println!("{}", ret_msg); - return Err(NotTargetInvoker(ret_msg).into()); - }); - } - - return Box::pin(async move { return Err(NoResponseErr(method_name).into()) }); - } - } - - #[tokio::test] - async fn test_failover_cluster() { - let mut cluster = FailoverCluster::new(Box::new(MockDirectory)); - let cluster = cluster.ready().await; - assert!(cluster.is_ok()); - - let cluster = cluster.unwrap(); - - let empty_stream = futures::stream::empty(); - let cloned_body = ClonedBody::new(empty_stream); - - let rpc_inv = RpcInvocation::default() - .with_service_unique_name("failover_cluster_service".to_owned()) - .with_method_name("invoker_request".to_owned()); - - let req = http::Request::builder() - .extension(rpc_inv) - .body(cloned_body) - .unwrap(); - - let ret = cluster.call(req).await; - assert!(ret.is_ok()); - - let ret = ret.unwrap(); - - assert_eq!(ret.status(), StatusCode::OK); + type Response = S::Response; - let body = ret.into_body(); + type Error = S::Error; - let mut pin = Box::pin(body); - let data = poll_fn(|cx| pin.as_mut().poll_data(cx)).await; - assert!(data.is_some()); - let data = data.unwrap(); - assert!(data.is_ok()); - let data = data.unwrap(); + type Future = S::Future; - assert_eq!( - String::from_utf8(data.to_vec()).unwrap(), - "invoker response" - ) + fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> std::task::Poll> { + self.inner.poll_ready(cx) } - - #[tokio::test] - async fn test_failover_request() { - let mut cluster = FailoverCluster::new(Box::new(MockDirectory)); - let cluster = cluster.ready().await; - assert!(cluster.is_ok()); - - let cluster = cluster.unwrap(); - - let once_stream = futures::stream::once(async { - let mut mut_bytes = BytesMut::default(); - mut_bytes.put_u8(5); - return Ok(mut_bytes.freeze()); - }); - let cloned_body = ClonedBody::new(once_stream); - - let rpc_inv = RpcInvocation::default() - .with_service_unique_name("failover_cluster_service".to_owned()) - .with_method_name("failover_request".to_owned()); - - let req = http::Request::builder() - .extension(rpc_inv) - .body(cloned_body) - .unwrap(); - - let ret = cluster.call(req).await; - assert!(ret.is_ok()); - - let ret = ret.unwrap(); - - assert_eq!(ret.status(), StatusCode::OK); - - let body = ret.into_body(); - - let mut pin = Box::pin(body); - let data = poll_fn(|cx| pin.as_mut().poll_data(cx)).await; - assert!(data.is_some()); - let data = data.unwrap(); - assert!(data.is_ok()); - let data = data.unwrap(); - - let resp_str = String::from_utf8(data.to_vec()).unwrap(); - println!("{}", resp_str); - assert_eq!(resp_str, "failover cluster index: 5 was invoked") + + fn call(&mut self, req: Request) -> Self::Future { + let (parts, body) = req.into_parts(); + let clone_body = CloneBody::new(body); + let req = Request::from_parts(parts, clone_body); + self.inner.call(req) } } diff --git a/dubbo/src/cluster/router/condition/condition_router.rs b/dubbo/src/cluster/router/condition/condition_router.rs deleted file mode 100644 index f39cd1c7..00000000 --- a/dubbo/src/cluster/router/condition/condition_router.rs +++ /dev/null @@ -1,60 +0,0 @@ -use crate::{ - cluster::router::{condition::single_router::ConditionSingleRouter, Router}, - codegen::RpcInvocation, -}; -use dubbo_base::Url; -use std::{ - fmt::Debug, - sync::{Arc, RwLock}, -}; - -#[derive(Default, Debug, Clone)] -pub struct ConditionRouter { - //condition router for service scope - pub service_routers: Option>>, - //condition router for application scope - pub application_routers: Option>>, -} - -impl Router for ConditionRouter { - fn route(&self, invokers: Vec, url: Url, invo: Arc) -> Vec { - let mut invokers_result = invokers.clone(); - if let Some(routers) = self.application_routers.clone() { - for router in &routers.read().unwrap().routers { - invokers_result = router.route(invokers_result, url.clone(), invo.clone()) - } - } - if let Some(routers) = self.service_routers.clone() { - for router in &routers.read().unwrap().routers { - invokers_result = router.route(invokers_result, url.clone(), invo.clone()) - } - } - invokers_result - } -} - -impl ConditionRouter { - pub fn new( - service_routers: Option>>, - application_routers: Option>>, - ) -> Self { - Self { - service_routers, - application_routers, - } - } -} - -#[derive(Debug, Clone, Default)] -pub struct ConditionSingleRouters { - pub routers: Vec, -} - -impl ConditionSingleRouters { - pub fn new(routers: Vec) -> Self { - Self { routers } - } - pub fn is_null(&self) -> bool { - self.routers.is_empty() - } -} diff --git a/dubbo/src/cluster/router/condition/matcher.rs b/dubbo/src/cluster/router/condition/matcher.rs deleted file mode 100644 index 8c9177e8..00000000 --- a/dubbo/src/cluster/router/condition/matcher.rs +++ /dev/null @@ -1,94 +0,0 @@ -use crate::codegen::RpcInvocation; -use regex::Regex; -use std::{collections::HashSet, error::Error, option::Option, sync::Arc}; - -#[derive(Clone, Debug, Default)] -pub struct ConditionMatcher { - _key: String, - matches: HashSet, - mismatches: HashSet, -} - -impl ConditionMatcher { - pub fn new(_key: String) -> Self { - ConditionMatcher { - _key, - matches: HashSet::new(), - mismatches: HashSet::new(), - } - } - - pub fn is_match( - &self, - value: Option, - invocation: Arc, - _is_when: bool, - ) -> Result> { - match value { - None => { - // if key does not present in whichever of url, invocation or attachment based on the matcher type, then return false. - Ok(false) - } - Some(val) => { - if !self.matches.is_empty() && self.mismatches.is_empty() { - for match_ in self.matches.iter() { - if self.do_pattern_match(match_, &val, invocation.clone())? { - return Ok(true); - } - } - Ok(false) - } else if !self.mismatches.is_empty() && self.matches.is_empty() { - for mismatch in self.mismatches.iter() { - if self.do_pattern_match(mismatch, &val, invocation.clone())? { - return Ok(false); - } - } - Ok(true) - } else if !self.matches.is_empty() && !self.mismatches.is_empty() { - for mismatch in self.mismatches.iter() { - if self.do_pattern_match(mismatch, &val, invocation.clone())? { - return Ok(false); - } - } - for match_ in self.matches.iter() { - if self.do_pattern_match(match_, &val, invocation.clone())? { - return Ok(true); - } - } - Ok(false) - } else { - Ok(false) - } - } - } - } - pub fn get_matches(&mut self) -> &mut HashSet { - &mut self.matches - } - pub fn get_mismatches(&mut self) -> &mut HashSet { - &mut self.mismatches - } - - fn do_pattern_match( - &self, - pattern: &String, - value: &String, - _invocation: Arc, - ) -> Result> { - if pattern.contains("*") { - return Ok(star_matcher(pattern, value)); - } - Ok(pattern.eq(value)) - } -} - -pub fn star_matcher(pattern: &str, input: &str) -> bool { - // 将*替换为任意字符的正则表达式 - let pattern = pattern.replace("*", ".*"); - let regex = Regex::new(&pattern).unwrap(); - regex.is_match(input) -} - -pub fn _range_matcher(val: i32, min: i32, max: i32) -> bool { - min <= val && val <= max -} diff --git a/dubbo/src/cluster/router/condition/mod.rs b/dubbo/src/cluster/router/condition/mod.rs deleted file mode 100644 index 7285b88f..00000000 --- a/dubbo/src/cluster/router/condition/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod condition_router; -pub mod matcher; -pub mod single_router; diff --git a/dubbo/src/cluster/router/condition/single_router.rs b/dubbo/src/cluster/router/condition/single_router.rs deleted file mode 100644 index e290b150..00000000 --- a/dubbo/src/cluster/router/condition/single_router.rs +++ /dev/null @@ -1,225 +0,0 @@ -use dubbo_base::Url; -use dubbo_logger::tracing::info; -use regex::Regex; -use std::{ - collections::HashMap, - sync::{Arc, RwLock}, -}; - -use crate::{ - cluster::router::{condition::matcher::ConditionMatcher, utils::to_original_map, Router}, - codegen::RpcInvocation, - invocation::Invocation, -}; - -#[derive(Debug, Clone, Default)] -pub struct ConditionSingleRouter { - pub name: String, - pub when_condition: HashMap>>, - pub then_condition: HashMap>>, - pub enabled: bool, - pub force: bool, -} - -impl Router for ConditionSingleRouter { - fn route(&self, invokers: Vec, url: Url, invocation: Arc) -> Vec { - if !self.enabled { - return invokers; - }; - let mut result = Vec::new(); - if self.match_when(url.clone(), invocation.clone()) { - for invoker in &invokers.clone() { - if self.match_then(invoker.clone(), invocation.clone()) { - result.push(invoker.clone()); - } - } - if result.is_empty() && self.force == false { - invokers - } else { - result - } - } else { - invokers - } - } -} - -impl ConditionSingleRouter { - pub fn new(rule: String, force: bool, enabled: bool) -> Self { - let mut router = Self { - name: "condition".to_string(), - when_condition: HashMap::new(), - then_condition: HashMap::new(), - enabled, - force, - }; - if enabled { - router.init(rule).expect("parse rule error"); - } - router - } - - fn init(&mut self, rule: String) -> Result<(), Box> { - match rule.trim().is_empty() { - true => Err("Illegal route rule!".into()), - false => { - let r = rule.replace("consumer.", "").replace("provider.", ""); - let i = r.find("=>").unwrap_or_else(|| r.len()); - let when_rule = r[..i].trim().to_string(); - let then_rule = r[(i + 2)..].trim().to_string(); - let when = if when_rule.is_empty() || when_rule == "true" { - HashMap::new() - } else { - self.parse_rule(&when_rule)? - }; - let then = if then_rule.is_empty() || then_rule == "false" { - HashMap::new() - } else { - self.parse_rule(&then_rule)? - }; - self.when_condition = when; - self.then_condition = then; - Ok(()) - } - } - } - - fn parse_rule( - &mut self, - rule: &str, - ) -> Result>>, Box> { - let mut conditions: HashMap>> = HashMap::new(); - let mut current_matcher: Option>> = None; - let regex = Regex::new(r"([&!=,]*)\s*([^&!=,\s]+)").unwrap(); - for cap in regex.captures_iter(rule) { - let separator = &cap[1]; - let content = &cap[2]; - - match separator { - "" => { - let current_key = content.to_string(); - current_matcher = - Some(Arc::new(RwLock::new(self.get_matcher(current_key.clone())))); - conditions.insert( - current_key.clone(), - current_matcher.as_ref().unwrap().clone(), - ); - } - "&" => { - let current_key = content.to_string(); - current_matcher = conditions.get(¤t_key).cloned(); - if current_matcher.is_none() { - let matcher = Arc::new(RwLock::new(self.get_matcher(current_key.clone()))); - conditions.insert(current_key.clone(), matcher.clone()); - current_matcher = Some(matcher); - } - } - "=" => { - if let Some(matcher) = ¤t_matcher { - let mut matcher_borrowed = matcher.write().unwrap(); - matcher_borrowed - .get_matches() - .insert(content.to_string().clone()); - } else { - return Err("Error: ...".into()); - } - } - "!=" => { - if let Some(matcher) = ¤t_matcher { - let mut matcher_borrowed = matcher.write().unwrap(); - matcher_borrowed - .get_mismatches() - .insert(content.to_string().clone()); - } else { - return Err("Error: ...".into()); - } - } - "," => { - if let Some(matcher) = ¤t_matcher { - let mut matcher_borrowed = matcher.write().unwrap(); - if matcher_borrowed.get_matches().is_empty() - && matcher_borrowed.get_mismatches().is_empty() - { - return Err("Error: ...".into()); - } - drop(matcher_borrowed); - let mut matcher_borrowed_mut = matcher.write().unwrap(); - matcher_borrowed_mut - .get_matches() - .insert(content.to_string().clone()); - } else { - return Err("Error: ...".into()); - } - } - _ => { - return Err("Error: ...".into()); - } - } - } - Ok(conditions) - } - - // pub fn is_runtime(&self) -> bool { - // // same as the Java version - // } - - pub fn get_matcher(&self, key: String) -> ConditionMatcher { - ConditionMatcher::new(key) - } - - pub fn match_when(&self, url: Url, invocation: Arc) -> bool { - if self.when_condition.is_empty() { - true - } else { - false - }; - self.do_match(url, &self.when_condition, invocation, true) - } - - pub fn match_then(&self, url: Url, invocation: Arc) -> bool { - if self.when_condition.is_empty() { - true - } else { - false - }; - self.do_match(url, &self.then_condition, invocation, false) - } - - pub fn do_match( - &self, - url: Url, - conditions: &HashMap>>, - invocation: Arc, - is_when: bool, - ) -> bool { - let sample: HashMap = to_original_map(url); - for (key, condition_matcher) in conditions { - let matcher = condition_matcher.read().unwrap(); - let value = get_value(key, &sample, invocation.clone()); - match matcher.is_match(value, invocation.clone(), is_when) { - Ok(result) => { - if !result { - return false; - } - } - Err(error) => { - info!("Error occurred: {:?}", error); - return false; - } - } - } - true - } -} - -fn get_value( - key: &String, - sample: &HashMap, - invocation: Arc, -) -> Option { - if key == "method" { - let method_param = invocation.get_method_name(); - return Some(method_param); - } - sample.get(key).cloned() -} diff --git a/dubbo/src/cluster/router/manager/condition_manager.rs b/dubbo/src/cluster/router/manager/condition_manager.rs deleted file mode 100644 index 4207b79b..00000000 --- a/dubbo/src/cluster/router/manager/condition_manager.rs +++ /dev/null @@ -1,75 +0,0 @@ -use crate::cluster::router::condition::{ - condition_router::{ConditionRouter, ConditionSingleRouters}, - single_router::ConditionSingleRouter, -}; -use dubbo_config::router::ConditionRouterConfig; -use std::{ - collections::HashMap, - sync::{Arc, RwLock}, -}; - -#[derive(Debug, Clone, Default)] -pub struct ConditionRouterManager { - //Application-level routing applies globally, while service-level routing only affects a specific service. - pub routers_service: HashMap>>, - pub routers_application: Arc>, -} - -impl ConditionRouterManager { - pub fn get_router(&self, service_name: String) -> Option { - let routers_service = self.routers_service.get(&service_name); - match routers_service { - Some(routers_service) => { - if self.routers_application.read().unwrap().is_null() { - return Some(ConditionRouter::new(Some(routers_service.clone()), None)); - } - Some(ConditionRouter::new( - Some(routers_service.clone()), - Some(self.routers_application.clone()), - )) - } - None => { - if self.routers_application.read().unwrap().is_null() { - return None; - } - Some(ConditionRouter::new( - None, - Some(self.routers_application.clone()), - )) - } - } - } - - pub fn update(&mut self, config: ConditionRouterConfig) { - let force = config.force; - let scope = config.scope; - let key = config.key; - let enable = config.enabled; - let mut routers = Vec::new(); - for condition in config.conditions { - routers.push(ConditionSingleRouter::new(condition, force, enable)); - } - match scope.as_str() { - "application" => { - self.routers_application.write().unwrap().routers = routers; - } - "service" => { - if let Some(x) = self.routers_service.get(&key) { - x.write().unwrap().routers = routers - } else { - self.routers_service.insert( - key, - Arc::new(RwLock::new(ConditionSingleRouters::new(routers))), - ); - } - } - _ => {} - } - } - - pub fn _parse_rules(&mut self, configs: Vec) { - for config in configs { - self.update(config) - } - } -} diff --git a/dubbo/src/cluster/router/manager/mod.rs b/dubbo/src/cluster/router/manager/mod.rs deleted file mode 100644 index 025f6c16..00000000 --- a/dubbo/src/cluster/router/manager/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -mod condition_manager; -pub mod router_manager; -mod tag_manager; diff --git a/dubbo/src/cluster/router/manager/router_manager.rs b/dubbo/src/cluster/router/manager/router_manager.rs deleted file mode 100644 index a2a66586..00000000 --- a/dubbo/src/cluster/router/manager/router_manager.rs +++ /dev/null @@ -1,161 +0,0 @@ -use crate::{ - cluster::router::{ - manager::{condition_manager::ConditionRouterManager, tag_manager::TagRouterManager}, - nacos_config_center::nacos_client::NacosClient, - router_chain::RouterChain, - }, - invocation::{Invocation, RpcInvocation}, -}; -use dubbo_base::Url; -use dubbo_config::{ - get_global_config, - router::{ConditionRouterConfig, NacosConfig, TagRouterConfig}, -}; -use dubbo_logger::tracing::{info, trace}; -use once_cell::sync::OnceCell; -use std::{ - collections::HashMap, - sync::{Arc, RwLock}, -}; - -pub static GLOBAL_ROUTER_MANAGER: OnceCell>> = OnceCell::new(); - -pub struct RouterManager { - pub condition_router_manager: ConditionRouterManager, - pub tag_router_manager: TagRouterManager, - pub nacos: Option, - pub consumer: HashMap, -} - -impl RouterManager { - pub fn get_router_chain(&self, invocation: Arc) -> RouterChain { - let service = invocation.get_target_service_unique_name().clone(); - let condition_router = self.condition_router_manager.get_router(service.clone()); - let tag_router = self.tag_router_manager.get_router(); - let mut chain = RouterChain::new(); - match self.consumer.get(service.as_str()) { - None => {} - Some(url) => { - chain.set_condition_router(condition_router); - chain.set_tag_router(tag_router); - chain.self_url = url.clone(); - } - } - chain - } - - pub fn notify(&mut self, event: RouterConfigChangeEvent) { - match event.router_kind.as_str() { - "condition" => { - let config: ConditionRouterConfig = - serde_yaml::from_str(event.content.as_str()).unwrap(); - self.condition_router_manager.update(config) - } - "tag" => { - let config: TagRouterConfig = serde_yaml::from_str(event.content.as_str()).unwrap(); - self.tag_router_manager.update(config) - } - _ => { - info!("other router change event") - } - } - } - - pub fn init_nacos(&mut self, config: NacosConfig) { - self.nacos = Some(NacosClient::new_init_client(config)); - self.init_router_managers_for_nacos(); - } - - pub fn init_router_managers_for_nacos(&mut self) { - let config = self - .nacos - .as_ref() - .unwrap() - .get_tag_config("application".to_string()); - match config { - None => {} - Some(tag_config) => { - self.tag_router_manager.init(); - self.tag_router_manager.update(tag_config) - } - } - for (service_name, _) in &self.consumer { - let config = self - .nacos - .as_ref() - .unwrap() - .get_condition_config(service_name.clone()); - match config { - None => {} - Some(condition_config) => self.condition_router_manager.update(condition_config), - } - } - } - - pub fn init(&mut self) { - let config = get_global_config().routers.clone(); - let consumer_configs = get_global_config() - .routers - .consumer - .clone() - .unwrap_or(Vec::new()); - for consumer_config in consumer_configs { - self.consumer.insert( - consumer_config.service.clone(), - Url::from_url( - format!("{}/{}", consumer_config.url, consumer_config.service).as_str(), - ) - .expect("consumer配置出错！Url生成错误"), - ); - } - match &config.nacos { - None => { - trace!("Nacos not configured, using local YAML configuration for routing"); - let condition = config.conditions.clone(); - match condition { - None => { - info!("Unconfigured Condition Router") - } - Some(cons) => { - for con in cons { - self.condition_router_manager.update(con) - } - } - } - let tag = config.tags.clone(); - match tag { - None => { - info!("Unconfigured Tag Router") - } - Some(ta) => { - self.tag_router_manager.init(); - self.tag_router_manager.update(ta) - } - } - } - Some(config) => { - self.init_nacos(config.clone()); - } - } - } -} - -pub fn get_global_router_manager() -> &'static Arc> { - GLOBAL_ROUTER_MANAGER.get_or_init(|| { - let mut router_manager = RouterManager { - condition_router_manager: ConditionRouterManager::default(), - tag_router_manager: TagRouterManager::default(), - nacos: None, - consumer: HashMap::new(), - }; - router_manager.init(); - return Arc::new(RwLock::new(router_manager)); - }) -} - -#[derive(Debug, Default, Clone)] -pub struct RouterConfigChangeEvent { - pub service_name: String, - pub router_kind: String, - pub content: String, -} diff --git a/dubbo/src/cluster/router/manager/tag_manager.rs b/dubbo/src/cluster/router/manager/tag_manager.rs deleted file mode 100644 index ce30c92e..00000000 --- a/dubbo/src/cluster/router/manager/tag_manager.rs +++ /dev/null @@ -1,27 +0,0 @@ -use crate::cluster::router::tag::tag_router::TagRouter; -use dubbo_config::router::TagRouterConfig; -use std::sync::{Arc, RwLock}; - -#[derive(Debug, Clone, Default)] -pub struct TagRouterManager { - pub tag_router: Option>>, -} - -impl TagRouterManager { - pub fn init(&mut self) { - self.tag_router = Some(Arc::new(RwLock::new(TagRouter::default()))) - } - - pub fn get_router(&self) -> Option>> { - self.tag_router.clone() - } - - pub fn update(&mut self, config: TagRouterConfig) { - self.tag_router - .as_ref() - .unwrap() - .write() - .unwrap() - .parse_config(config) - } -} diff --git a/dubbo/src/cluster/router/mod.rs b/dubbo/src/cluster/router/mod.rs deleted file mode 100644 index 84ceae15..00000000 --- a/dubbo/src/cluster/router/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -pub mod condition; -pub mod manager; -pub mod nacos_config_center; -pub mod router_chain; -pub mod tag; -pub mod utils; - -use crate::invocation::RpcInvocation; -use dubbo_base::Url; -use std::{fmt::Debug, sync::Arc}; - -pub trait Router: Debug + Clone { - fn route(&self, invokers: Vec, url: Url, invo: Arc) -> Vec; -} - -// pub type BoxRouter = Box; - -#[derive(Debug, Default, Clone)] -pub struct MockRouter {} - -impl Router for MockRouter { - fn route(&self, invokers: Vec, _url: Url, _invo: Arc) -> Vec { - invokers - } -} diff --git a/dubbo/src/cluster/router/nacos_config_center/mod.rs b/dubbo/src/cluster/router/nacos_config_center/mod.rs deleted file mode 100644 index 7878fa9f..00000000 --- a/dubbo/src/cluster/router/nacos_config_center/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod nacos_client; diff --git a/dubbo/src/cluster/router/nacos_config_center/nacos_client.rs b/dubbo/src/cluster/router/nacos_config_center/nacos_client.rs deleted file mode 100644 index 1600e9b0..00000000 --- a/dubbo/src/cluster/router/nacos_config_center/nacos_client.rs +++ /dev/null @@ -1,132 +0,0 @@ -use crate::cluster::router::manager::router_manager::{ - get_global_router_manager, RouterConfigChangeEvent, -}; -use dubbo_config::router::{ConditionRouterConfig, NacosConfig, TagRouterConfig}; -use dubbo_logger::{tracing, tracing::info}; -use nacos_sdk::api::{ - config::{ConfigChangeListener, ConfigResponse, ConfigService, ConfigServiceBuilder}, - props::ClientProps, -}; -use std::sync::{Arc, RwLock}; - -pub struct NacosClient { - pub client: Arc>, -} - -unsafe impl Send for NacosClient {} - -unsafe impl Sync for NacosClient {} - -pub struct ConfigChangeListenerImpl; - -impl NacosClient { - pub fn new_init_client(config: NacosConfig) -> Self { - let server_addr = config.addr; - let namespace = config.namespace; - let app = config.app; - match config.enable_auth { - None => { - info!("disable nacos auth!"); - info!("nacos init,addr:{}", server_addr); - let client = Arc::new(RwLock::new( - ConfigServiceBuilder::new( - ClientProps::new() - .server_addr(server_addr) - .namespace(namespace) - .app_name(app), - ) - .build() - .expect("NacosClient build failed!Please check NacosConfig"), - )); - Self { client } - } - Some(auth) => { - info!("enable nacos auth!"); - info!("nacos init,addr:{}", server_addr); - let client = Arc::new(RwLock::new( - ConfigServiceBuilder::new( - ClientProps::new() - .server_addr(server_addr) - .namespace(namespace) - .app_name(app) - .auth_username(auth.auth_username) - .auth_password(auth.auth_password), - ) - // .enable_auth_plugin_http() - .build() - .expect("NacosClient build failed!Please check NacosConfig"), - )); - return Self { client }; - } - } - } - pub fn get_condition_config(&self, data_id: String) -> Option { - let config_resp = self - .client - .read() - .unwrap() - .get_config(data_id.clone(), "condition".to_string()); - return match config_resp { - Ok(config_resp) => { - self.add_listener(data_id.clone(), "condition".to_string()); - let string = config_resp.content(); - Some(serde_yaml::from_str(string).unwrap()) - } - Err(_err) => None, - }; - } - - pub fn get_tag_config(&self, data_id: String) -> Option { - let config_resp = self - .client - .read() - .unwrap() - .get_config(data_id.clone(), "tag".to_string()); - return match config_resp { - Ok(config_resp) => { - self.add_listener(data_id.clone(), "tag".to_string()); - let string = config_resp.content(); - let result = serde_yaml::from_str(string); - match result { - Ok(config) => { - info!("success to get TagRouter config and parse success"); - Some(config) - } - _ => { - info!("failed to parse TagRouter rule"); - None - } - } - } - Err(_err) => None, - }; - } - pub fn add_listener(&self, data_id: String, group: String) { - let res_listener = self - .client - .write() - .expect("failed to create nacos config listener") - .add_listener(data_id, group, Arc::new(ConfigChangeListenerImpl {})); - match res_listener { - Ok(_) => { - info!("listening the config success"); - } - Err(err) => tracing::error!("listen config error {:?}", err), - } - } -} - -impl ConfigChangeListener for ConfigChangeListenerImpl { - fn notify(&self, config_resp: ConfigResponse) { - let content_type = config_resp.content_type(); - let event = RouterConfigChangeEvent { - service_name: config_resp.data_id().clone(), - router_kind: config_resp.group().clone(), - content: config_resp.content().clone(), - }; - if content_type == "yaml" { - get_global_router_manager().write().unwrap().notify(event); - } - info!("notify config={:?}", config_resp); - } -} diff --git a/dubbo/src/cluster/router/router_chain.rs b/dubbo/src/cluster/router/router_chain.rs deleted file mode 100644 index 930f0f29..00000000 --- a/dubbo/src/cluster/router/router_chain.rs +++ /dev/null @@ -1,74 +0,0 @@ -use crate::{ - cluster::router::{ - condition::condition_router::ConditionRouter, tag::tag_router::TagRouter, Router, - }, - invocation::RpcInvocation, -}; -use dubbo_base::Url; -use std::sync::{Arc, RwLock}; - -#[derive(Debug, Default, Clone)] -pub struct RouterChain { - pub condition_router: Option, - pub tag_router: Option>>, - pub self_url: Url, -} - -impl RouterChain { - pub fn new() -> Self { - RouterChain { - condition_router: None, - tag_router: None, - self_url: Url::new(), - } - } - pub fn set_condition_router(&mut self, router: Option) { - self.condition_router = router; - } - pub fn set_tag_router(&mut self, router: Option>>) { - self.tag_router = router; - } - pub fn route(&self, invokers: Vec, invocation: Arc) -> Vec { - let mut result = invokers.clone(); - match &self.tag_router { - None => {} - Some(router) => { - result = - router - .read() - .unwrap() - .route(result, self.self_url.clone(), invocation.clone()) - } - } - match &self.condition_router { - Some(router) => { - result = router.route(result, self.self_url.clone(), invocation.clone()) - } - None => {} - } - result - } -} - -#[test] -fn test() { - use crate::cluster::router::manager::router_manager::get_global_router_manager; - - let u1 = Url::from_url("triple://127.0.0.1:8888/org.apache.dubbo.sample.tri.Greeter").unwrap(); - let u2 = Url::from_url("triple://127.0.0.1:8889/org.apache.dubbo.sample.tri.Greeter").unwrap(); - let u3 = Url::from_url("triple://127.0.0.1:8800/org.apache.dubbo.sample.tri.Greeter").unwrap(); - let u4 = Url::from_url("triple://127.0.2.1:880/org.apache.dubbo.sample.tri.Greeter").unwrap(); - let u5 = Url::from_url("triple://127.0.1.1:8888/org.apache.dubbo.sample.tri.Greeter").unwrap(); - let invos = vec![u1, u2, u3, u4, u5]; - let invo = Arc::new( - RpcInvocation::default() - .with_method_name("greet".to_string()) - .with_service_unique_name("org.apache.dubbo.sample.tri.Greeter".to_string()), - ); - let x = get_global_router_manager() - .read() - .unwrap() - .get_router_chain(invo.clone()); - let result = x.route(invos, invo.clone()); - dbg!(result); -} diff --git a/dubbo/src/cluster/router/tag/mod.rs b/dubbo/src/cluster/router/tag/mod.rs deleted file mode 100644 index 6ac5b218..00000000 --- a/dubbo/src/cluster/router/tag/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod tag_router; diff --git a/dubbo/src/cluster/router/tag/tag_router.rs b/dubbo/src/cluster/router/tag/tag_router.rs deleted file mode 100644 index d1a962e7..00000000 --- a/dubbo/src/cluster/router/tag/tag_router.rs +++ /dev/null @@ -1,70 +0,0 @@ -use crate::{ - cluster::router::{utils::to_original_map, Router}, - codegen::RpcInvocation, -}; -use dubbo_base::Url; -use dubbo_config::router::TagRouterConfig; -use std::{collections::HashMap, fmt::Debug, sync::Arc}; - -#[derive(Debug, Clone, Default)] -pub struct TagRouter { - pub tag_rules: HashMap>, - pub force: bool, - pub enabled: bool, -} - -impl TagRouter { - pub fn parse_config(&mut self, config: TagRouterConfig) { - self.tag_rules = HashMap::new(); - self.force = config.force; - self.enabled = config.enabled; - for tag in &config.tags { - let mut tags = HashMap::new(); - for rule in &tag.matches { - tags.insert(rule.key.clone(), rule.value.clone()); - } - self.tag_rules.insert(tag.name.clone(), tags); - } - } - - pub fn match_tag(&self, params: HashMap) -> Option { - let mut tag_result = None; - for (tag, tag_rules) in &self.tag_rules { - for (key, value) in tag_rules { - match params.get(key.as_str()) { - None => {} - Some(val) => { - if val == value { - tag_result = Some(tag.clone()) - } - } - } - } - } - tag_result - } -} - -impl Router for TagRouter { - fn route(&self, invokers: Vec, url: Url, _invocation: Arc) -> Vec { - if !self.enabled { - return invokers; - }; - let self_param = to_original_map(url); - let invocation_tag = self.match_tag(self_param); - let mut invokers_result = Vec::new(); - for invoker in &invokers { - let invoker_param = to_original_map(invoker.clone()); - let invoker_tag = self.match_tag(invoker_param); - if invoker_tag == invocation_tag { - invokers_result.push(invoker.clone()) - } - } - if invokers_result.is_empty() { - if !self.force { - return invokers; - } - } - invokers_result - } -} diff --git a/dubbo/src/cluster/router/utils.rs b/dubbo/src/cluster/router/utils.rs deleted file mode 100644 index 2ca50fcc..00000000 --- a/dubbo/src/cluster/router/utils.rs +++ /dev/null @@ -1,16 +0,0 @@ -use dubbo_base::Url; -use std::{collections::HashMap, string::String}; - -pub fn to_original_map(url: Url) -> HashMap { - let mut result: HashMap = HashMap::new(); - result.insert("scheme".parse().unwrap(), url.scheme); - result.insert("location".parse().unwrap(), url.location); - result.insert("ip".parse().unwrap(), url.ip); - result.insert("port".parse().unwrap(), url.port); - result.insert("service_name".parse().unwrap(), url.service_name); - result.insert("service_key".parse().unwrap(), url.service_key); - for (key, value) in url.params { - result.insert(key, value); - } - result -} diff --git a/dubbo/src/codegen.rs b/dubbo/src/codegen.rs index 412feb91..38f8f1dc 100644 --- a/dubbo/src/codegen.rs +++ b/dubbo/src/codegen.rs @@ -27,11 +27,9 @@ pub use hyper::Body as hyperBody; pub use tower_service::Service; pub use super::{ - cluster::directory::RegistryDirectory, empty_body, invocation::{IntoStreamingRequest, Request, Response, RpcInvocation}, protocol::{triple::triple_invoker::TripleInvoker, Invoker}, - registry::{BoxRegistry, Registry}, triple::{ client::TripleClient, codec::{prost::ProstCodec, Codec}, @@ -46,7 +44,7 @@ pub use super::{ pub use crate::{ filter::{service::FilterService, Filter}, triple::{ - client::builder::{ClientBoxService, ClientBuilder}, + client::builder::ClientBuilder, server::builder::ServerBuilder, transport::connection::Connection, }, diff --git a/dubbo/src/directory/mod.rs b/dubbo/src/directory/mod.rs new file mode 100644 index 00000000..85a463e6 --- /dev/null +++ b/dubbo/src/directory/mod.rs @@ -0,0 +1,323 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + use core::panic; +use std::{ + hash::Hash, + pin::Pin, + task::{Context, Poll}, collections::HashMap, sync::{Arc, Mutex}, marker::PhantomData, +}; + +use crate::{StdError, codegen::RpcInvocation, invocation::Invocation}; +use dubbo_base::{param::Param, svc::NewService}; +use futures_core::Future; +use futures_util::future::{poll_fn, self}; +use pin_project::pin_project; +use tokio::{sync::{watch, Notify}, select}; +use tower::{ + discover::{Change, Discover}, ServiceExt, buffer::Buffer, util::{FutureService, Oneshot}, +}; + +use tower_service::Service; + +pub struct NewCachedDirectory(PhantomData); + + +pub struct CachedDirectory +where + // NewDirectory + N: NewService +{ + inner: N, + cache: Arc>> +} + + +pub struct NewDirectory { + // service registry + inner: N, +} + + +#[pin_project] +pub struct NewDirectoryFuture +where + S: Service<()>, + S::Response: Discover +{ + #[pin] + inner: Oneshot +} + +#[derive(Clone)] +pub struct Directory +where + D: Discover +{ + rx: watch::Receiver>, + close: Arc +} + +impl NewCachedDirectory { + + pub fn new() -> Self { + NewCachedDirectory(PhantomData) + } +} + + +impl NewService for NewCachedDirectory +where + T: Param, + // service registry + N: NewService, + N::Service: Service<()> + Send, + // Discover service + >::Response: Discover + Unpin + Send, + >::Error: Into, + >::Future: Unpin + Send, + // Discover::Key + <>::Response as Discover>::Key: Hash + Eq + Clone + Send, + // Discover::Service = new invoker + <>::Response as Discover>::Service: NewService<()> + Clone + Send + Sync, +{ + type Service = CachedDirectory, T>; + + fn new_service(&self, target: N) -> Self::Service { + + CachedDirectory::new(NewDirectory::new(target)) + } +} + + +impl CachedDirectory +where + N: NewService +{ + + pub fn new(inner: N) -> Self { + CachedDirectory { + inner, + cache: Default::default() + } + } +} + + +impl NewService for CachedDirectory +where + T: Param, + // NewDirectory + N: NewService, + // Buffered directory + N::Service: Clone +{ + type Service = N::Service; + + fn new_service(&self, target: T) -> Self::Service { + let rpc_invocation = target.param(); + let service_name = rpc_invocation.get_target_service_unique_name(); + let mut cache = self.cache.lock().expect("cached directory lock failed."); + let value = cache.get(&service_name).map(|val|val.clone()); + match value { + None => { + let new_service = self.inner.new_service(target); + cache.insert(service_name, new_service.clone()); + new_service + }, + Some(value) => value + } + } +} + + +impl NewDirectory { + + pub fn new(inner: N) -> Self { + NewDirectory { + inner + } + } +} + +impl NewService for NewDirectory +where + T: Param, + // service registry + N: NewService, + N::Service: Service<()> + Send, + // Discover service + >::Response: Discover + Unpin + Send, + >::Future: Unpin + Send, + >::Error: Into, + // Discover::Key + <>::Response as Discover>::Key: Hash + Eq + Clone + Send, + // Discover::service = new invoker + <>::Response as Discover>::Service: Send + Sync + Clone + NewService<()>, +{ + type Service = Buffer>::Service>, Directory<>::Response>>, ()>; + + fn new_service(&self, target: T) -> Self::Service { + let discovery_service = self.inner.new_service(target); + Buffer::new(FutureService::new(NewDirectoryFuture::new(discovery_service)), 10) + } + +} + + + +impl NewDirectoryFuture +where + // Discover service + S: Service<()>, + S::Response: Discover +{ + pub fn new(inner: S) -> Self { + NewDirectoryFuture { + inner: inner.oneshot(()) + } + } +} + + + +impl Future for NewDirectoryFuture +where + // Discover service + S: Service<()>, + // Discover + S::Response: Discover + Unpin + Send, + // the key may be dubbo url + ::Key: Hash + Eq + Clone + Send, + // error + S::Error: Into, + // new invoker + ::Service: NewService<()> + Clone + Send + Sync +{ + type Output = Result, StdError>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.project(); + + this.inner.poll(cx).map(|poll_ret| { + poll_ret.map(|discover| { + Directory::new(discover) + }) + }).map_err(|e| { + e.into() + }) + + } +} + +impl Directory +where + // Discover + D: Discover + Unpin + Send, + // the key may be dubbo url + D::Key: Hash + Eq + Clone + Send, + // invoker new service + D::Service: NewService<()> + Clone + Send + Sync, +{ + + pub fn new(mut discover: D) -> Self { + + let pin_discover = Pin::new(&mut discover); + let (tx, rx) = watch::channel(Vec::new()); + let close = Arc::new(Notify::new()); + let close_clone = close.clone(); + + tokio::spawn(async move { + let mut cache: HashMap = HashMap::new(); + + loop { + let changed = select! { + _ = close_clone.notified() => { + // info!("discover stream closed.") + return; + }, + changed = poll_fn(|cx| pin_discover.poll_discover(cx)) => { + changed + } + }; + let Some(changed) = changed else { + // debug!("discover stream closed."); + break; + }; + + match changed { + Err(e) => { + // error!("discover stream error: {}", e); + continue; + }, + Ok(changed) => match changed { + Change::Insert(k, v) => { + cache.insert(k, v); + }, + Change::Remove(k) => { + cache.remove(&k); + } + } + } + + let vec: Vec = cache.values().map(|v|v.clone()).collect(); + let _ = tx.send(vec); + } + + }); + Directory { + rx, + close + } + } +} + + +impl Service<()> for Directory +where + // Discover + D: Discover + Unpin + Send, + // the key may be dubbo url + D::Key: Hash + Eq + Clone + Send, + // invoker new service + D::Service: NewService<()> + Clone + Send + Sync, +{ + type Response = watch::Receiver>; + + type Error = StdError; + + type Future = future::Ready>; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, _: ()) -> Self::Future { + future::ok(self.rx.clone()) + } +} + +impl Drop for Directory +where + D: Discover, +{ + fn drop(&mut self) { + if Arc::strong_count(&self.close) == 1 { + self.close.notify_one(); + } + } +} \ No newline at end of file diff --git a/dubbo/src/invocation.rs b/dubbo/src/invocation.rs index d3eb8ad0..57503379 100644 --- a/dubbo/src/invocation.rs +++ b/dubbo/src/invocation.rs @@ -196,7 +196,7 @@ pub trait Invocation { fn get_method_name(&self) -> String; } -#[derive(Default,Clone)] +#[derive(Default, Clone)] pub struct RpcInvocation { target_service_unique_name: String, method_name: String, @@ -224,4 +224,4 @@ impl Invocation for RpcInvocation { fn get_method_name(&self) -> String { self.method_name.clone() } -} \ No newline at end of file +} diff --git a/dubbo/src/lib.rs b/dubbo/src/lib.rs index 63c09d3a..c614d333 100644 --- a/dubbo/src/lib.rs +++ b/dubbo/src/lib.rs @@ -26,6 +26,9 @@ pub mod registry; pub mod status; pub mod triple; pub mod utils; +pub mod directory; +pub mod route; +pub mod loadbalancer; use http_body::Body; use std::{future::Future, pin::Pin}; diff --git a/dubbo/src/loadbalancer/mod.rs b/dubbo/src/loadbalancer/mod.rs new file mode 100644 index 00000000..d4feae75 --- /dev/null +++ b/dubbo/src/loadbalancer/mod.rs @@ -0,0 +1,103 @@ +use dubbo_base::{svc::NewService, param::Param}; +use futures_core::future::BoxFuture; +use tower::{discover::ServiceList, ServiceExt}; +use tower_service::Service; + +use crate::{codegen::RpcInvocation, StdError}; + +pub struct NewLoadBalancer { + inner: N +} + +#[derive(Clone)] +pub struct LoadBalancer { + inner: S, // Routes service +} + + +impl NewLoadBalancer { + + pub fn layer() -> impl tower_layer::Layer{ + + tower_layer::layer_fn(|inner| { + NewLoadBalancer { + inner // NewRoutes + } + }) + } +} + +impl NewService for NewLoadBalancer +where + T: Param + Clone, + // NewRoutes + N: NewService +{ + + type Service = LoadBalancer; + + fn new_service(&self, target: T) -> Self::Service { + // Routes service + let svc = self.inner.new_service(target); + + LoadBalancer { + inner: svc + } + + } +} + +impl Service for LoadBalancer +where + Req: Send + 'static, + // Routes service + N: Service<(), Response = Vec>, + N::Error: Into + Send, + N::Future: Send + 'static, + // new invoker + Nsv: NewService<()> + Send, + Nsv::Service: Service + Send, + // invoker + >::Error: Into + Send, + >::Future: Send + 'static, +{ + + type Response = >::Response; + + type Error = StdError; + + type Future = BoxFuture<'static, Result>; + + fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> std::task::Poll> { + self.inner.poll_ready(cx).map_err(Into::into) + + } + + fn call(&mut self, req: Req) -> Self::Future { + let routes = self.inner.call(()); + + let fut = async move { + let routes = routes.await; + + let routes: Vec = match routes { + Err(e) => return Err(Into::::into(e)), + Ok(routes) => routes + }; + + let service_list: Vec<_> = routes.iter().map(|inv| { + let invoker = inv.new_service(()); + tower::load::Constant::new(invoker, 1) + + }).collect(); + + let service_list = ServiceList::new(service_list); + + let p2c = tower::balance::p2c::Balance::new(service_list); + + + p2c.oneshot(req).await + }; + + Box::pin(fut) + } +} diff --git a/dubbo/src/protocol/mod.rs b/dubbo/src/protocol/mod.rs index 50080555..b7d67835 100644 --- a/dubbo/src/protocol/mod.rs +++ b/dubbo/src/protocol/mod.rs @@ -26,8 +26,6 @@ use tower_service::Service; use dubbo_base::Url; -use crate::triple::client::replay::{ClonedBytesStream, ClonedBody}; - pub mod server_desc; pub mod triple; @@ -44,18 +42,19 @@ pub trait Exporter { fn unexport(&self); } -pub trait Invoker: Debug + Service { +pub trait Invoker: Service { fn get_url(&self) -> Url; } pub type BoxExporter = Box; pub type BoxInvoker = Box< dyn Invoker< - http::Request, + http::Request, Response = http::Response, Error = crate::Error, Future = crate::BoxFuture, crate::Error>, - > + Send, + > + Send + + Sync, >; pub struct WrapperInvoker(T); diff --git a/dubbo/src/protocol/triple/triple_invoker.rs b/dubbo/src/protocol/triple/triple_invoker.rs index af174451..47867530 100644 --- a/dubbo/src/protocol/triple/triple_invoker.rs +++ b/dubbo/src/protocol/triple/triple_invoker.rs @@ -22,16 +22,12 @@ use std::{ }; use tower_service::Service; -use crate::{ - protocol::Invoker, - triple::{client::{builder::ClientBoxService, replay::ClonedBody}, transport::connection::Connection}, - utils::boxed_clone::BoxCloneService, -}; +use crate::triple::transport::connection::Connection; #[derive(Clone)] pub struct TripleInvoker { url: Url, - conn: ClientBoxService, + conn: Connection, } impl TripleInvoker { @@ -39,7 +35,7 @@ impl TripleInvoker { let uri = http::Uri::from_str(&url.to_url()).unwrap(); Self { url, - conn: BoxCloneService::new(Connection::new().with_host(uri)), + conn: Connection::new().with_host(uri), } } } @@ -50,14 +46,14 @@ impl Debug for TripleInvoker { } } -impl Service> for TripleInvoker { +impl Service> for TripleInvoker { type Response = http::Response; type Error = crate::Error; type Future = crate::BoxFuture; - fn call(&mut self, req: http::Request) -> Self::Future { + fn call(&mut self, req: http::Request) -> Self::Future { self.conn.call(req) } @@ -69,8 +65,3 @@ impl Service> for TripleInvoker { } } -impl Invoker> for TripleInvoker { - fn get_url(&self) -> Url { - self.url.clone() - } -} diff --git a/dubbo/src/registry/mod.rs b/dubbo/src/registry/mod.rs index 2a95452a..ab2297f3 100644 --- a/dubbo/src/registry/mod.rs +++ b/dubbo/src/registry/mod.rs @@ -20,6 +20,7 @@ pub mod integration; pub mod memory_registry; pub mod protocol; pub mod types; +pub mod n_registry; use std::{ fmt::{Debug, Formatter}, @@ -28,6 +29,7 @@ use std::{ use dubbo_base::Url; + pub type RegistryNotifyListener = Arc; pub trait Registry { fn register(&mut self, url: Url) -> Result<(), crate::StdError>; diff --git a/dubbo/src/registry/n_registry.rs b/dubbo/src/registry/n_registry.rs new file mode 100644 index 00000000..94d866c1 --- /dev/null +++ b/dubbo/src/registry/n_registry.rs @@ -0,0 +1,75 @@ +use async_trait::async_trait; +use dubbo_base::{Url, svc::NewService, param::Param}; +use tower::discover::Discover; +use tower_service::Service; + +use crate::{StdError, codegen::RpcInvocation, invocation::Invocation}; + +#[async_trait] +pub trait Registry { + + type Discover: Discover; + + type Error; + + fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> std::task::Poll>; + + async fn register(&self, url: Url) -> Result<(), Self::Error>; + + async fn unregister(&self, url: Url) -> Result<(), Self::Error>; + + // todo service_name change to url + async fn subscribe(&self, service_name: String) -> Result; + + async fn unsubscribe(&self, url: Url) -> Result<(), Self::Error>; +} + + +pub struct DiscoverService { + inner: N, + service_name: String, +} + +impl Service<()> for DiscoverService +where + N: Registry, + N::Discover: Discover + Unpin + Send, +{ + type Response = N::Discover; + type Error = StdError; + type Future = std::pin::Pin> + Send>>; + + fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> std::task::Poll> { + self.inner.poll_ready(cx) + } + + fn call(&mut self, req: ()) -> Self::Future { + let service_name = self.service_name.clone(); + let discover_fut = self.inner.subscribe(service_name); + Box::pin(async move { + discover_fut.await + }) + } + +} + + +impl NewService for R +where + T: Param, + R: Registry + Clone, + R::Discover: Discover + Unpin + Send, +{ + type Service = DiscoverService; + + fn new_service(&self, target: T) -> Self::Service { + let service_name = target.param().get_target_service_unique_name(); + + DiscoverService { + inner: self.clone(), + service_name, + } + } +} + + diff --git a/dubbo/src/route/mod.rs b/dubbo/src/route/mod.rs new file mode 100644 index 00000000..e701846d --- /dev/null +++ b/dubbo/src/route/mod.rs @@ -0,0 +1,135 @@ +use dubbo_base::{svc::NewService, param::Param}; +use futures_core::{Future, ready}; +use futures_util::future::Ready; +use pin_project::pin_project; +use tokio::{sync::watch, pin}; +use tower::util::FutureService; +use tower_service::Service; + +use crate::{StdError, codegen::RpcInvocation}; + +pub struct NewRoutes { + inner: N, +} + +#[pin_project] +pub struct NewRoutesFuture { + #[pin] + inner: N, + target: T, +} + +#[derive(Clone)] +pub struct Routes { + target: T, + new_invokers: Vec, + invokers_receiver: watch::Receiver>, +} + + +impl NewRoutes { + + pub fn layer() -> impl tower_layer::Layer { + tower_layer::layer_fn(|inner: N| { + NewRoutes { + inner, // NewDirectory + } + }) + } +} + + +impl NewService for NewRoutes +where + T: Param + Clone, + // NewDirectory + N: NewService, + // Directory + N::Service: Service<(), Response = watch::Receiver>>, + // new invoker service + Nsv: NewService<()> + Clone +{ + + type Service = FutureService, Routes>; + + fn new_service(&self, target: T) -> Self::Service { + let inner = self.inner.new_service(target.clone()); + + FutureService::new(NewRoutesFuture { + inner, + target, + }) + } +} + + +impl Future for NewRoutesFuture +where + T: Param + Clone, + // Directory + N: Service<(), Response = watch::Receiver>> + Unpin, + N::Error: Into, + // new invoker service + Nsv: NewService<()> + Clone, +{ + type Output = Result, StdError>; + + fn poll(self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll { + + let this = self.get_mut(); + + let target = this.target.clone(); + + let _ = ready!(this.inner.poll_ready(cx)).map_err(Into::into)?; + + + let call = this.inner.call(()); + pin!(call); + + let mut invokers_receiver = ready!(call.poll(cx).map_err(Into::into))?; + let new_invokers = { + let wait_for = invokers_receiver.wait_for(|invs|!invs.is_empty()); + pin!(wait_for); + + let changed = ready!(wait_for.poll(cx))?; + + changed.clone() + }; + + + std::task::Poll::Ready(Ok(Routes { + invokers_receiver, + new_invokers, + target, + })) + } +} + + + +impl Service<()> for Routes +where + T: Param + Clone, + // new invoker service + Nsv: NewService<()> + Clone, +{ + type Response = Vec; + + type Error = StdError; + + type Future = Ready>; + + fn poll_ready(&mut self, _: &mut std::task::Context<'_>) -> std::task::Poll> { + let has_change = self.invokers_receiver.has_changed()?; + if has_change { + self.new_invokers = self.invokers_receiver.borrow_and_update().clone(); + } + std::task::Poll::Ready(Ok(())) + } + + fn call(&mut self, _: ()) -> Self::Future { + // some router operator + // if new_invokers changed, send new invokers to routes_rx after router operator + futures_util::future::ok(self.new_invokers.clone()) + } +} \ No newline at end of file diff --git a/dubbo/src/triple/client/builder.rs b/dubbo/src/triple/client/builder.rs index 32b26a55..6f063f42 100644 --- a/dubbo/src/triple/client/builder.rs +++ b/dubbo/src/triple/client/builder.rs @@ -19,18 +19,17 @@ use std::sync::Arc; use crate::{ cluster::{directory::StaticDirectory, Cluster, Directory, MockCluster, MockDirectory}, - codegen::{RegistryDirectory, RpcInvocation, TripleInvoker}, + codegen::{RpcInvocation, TripleInvoker}, protocol::BoxInvoker, utils::boxed_clone::BoxCloneService, }; -use dubbo_base::Url; -use super::replay::ClonedBody; +use aws_smithy_http::body::SdkBody; +use dubbo_base::Url; pub type ClientBoxService = - BoxCloneService, http::Response, crate::Error>; + BoxCloneService, http::Response, crate::Error>; -#[allow(dead_code)] #[derive(Clone, Debug, Default)] pub struct ClientBuilder { pub timeout: Option, @@ -57,7 +56,7 @@ impl ClientBuilder { connector: "", directory: Some(Arc::new(Box::new(StaticDirectory::new(&host)))), direct: true, - host: host.to_string(), + host: host.clone().to_string(), } } @@ -101,7 +100,7 @@ impl ClientBuilder { Self { direct, ..self } } - pub fn build(self) -> Option { + pub fn build(self, _invocation: Arc) -> Option { if self.direct { return Some(Box::new(TripleInvoker::new( Url::from_url(&self.host).unwrap(), diff --git a/dubbo/src/triple/client/mod.rs b/dubbo/src/triple/client/mod.rs index 97be85eb..7a8e0131 100644 --- a/dubbo/src/triple/client/mod.rs +++ b/dubbo/src/triple/client/mod.rs @@ -17,5 +17,5 @@ pub mod builder; pub mod triple; -pub mod replay; + pub use triple::TripleClient; diff --git a/dubbo/src/triple/client/replay.rs b/dubbo/src/triple/client/replay.rs deleted file mode 100644 index 4ba8b930..00000000 --- a/dubbo/src/triple/client/replay.rs +++ /dev/null @@ -1,441 +0,0 @@ -use std::{ - collections::VecDeque, - pin::Pin, - sync::{Arc, Mutex}, - task::{Context, Poll}, -}; - -use bytes::{Buf, BufMut, Bytes, BytesMut}; -use futures_core::{ready, stream::BoxStream, Stream}; -use futures_util::StreamExt; -use http_body::Body; -use pin_project::pin_project; - -use crate::status::Status; - -type BoxBytesStream = BoxStream<'static, Result>; -pub struct ClonedBytesStream { - shared: Arc>>, - owned: Option, - replay: bool, -} - -pub struct OwnedBytesStream { - bytes_stream: BoxBytesStream, - buf: InnerBuffer, -} - -#[derive(Clone)] -pub struct InnerBuffer { - bufs: VecDeque, - capacity: usize, -} - -impl ClonedBytesStream { - pub fn new(buffer_capacity: usize, stream: BoxBytesStream) -> Self { - Self { - shared: Arc::new(Mutex::new(None)), - owned: Some(OwnedBytesStream { - bytes_stream: stream, - buf: InnerBuffer { - bufs: Default::default(), - capacity: buffer_capacity, - }, - }), - replay: false, - } - } -} - -impl Clone for ClonedBytesStream { - fn clone(&self) -> Self { - Self { - shared: self.shared.clone(), - owned: None, - replay: true, - } - } -} - -impl Drop for ClonedBytesStream { - fn drop(&mut self) { - if let Some(owned) = self.owned.take() { - let lock = self.shared.lock(); - if let Ok(mut lock) = lock { - *lock = Some(owned); - } - } - } -} - -impl InnerBuffer { - pub fn push_bytes(&mut self, bytes: Bytes) -> Bytes { - self.bufs.push_back(bytes.clone()); - bytes - } - - pub fn is_capped(&self) -> bool { - self.capacity == 0 - } -} - -impl Buf for InnerBuffer { - fn remaining(&self) -> usize { - self.bufs.iter().map(|bytes| bytes.remaining()).sum() - } - - fn chunk(&self) -> &[u8] { - self.bufs.front().map(Buf::chunk).unwrap_or(&[]) - } - - fn chunks_vectored<'a>(&'a self, dst: &mut [std::io::IoSlice<'a>]) -> usize { - if dst.is_empty() { - return 0; - } - - let mut filled = 0; - - for bytes in self.bufs.iter() { - filled += bytes.chunks_vectored(&mut dst[filled..]) - } - - filled - } - - fn advance(&mut self, mut cnt: usize) { - while cnt > 0 { - let first = self.bufs.front_mut(); - if first.is_none() { - break; - } - let first = first.unwrap(); - let first_remaining = first.remaining(); - if first_remaining > cnt { - first.advance(cnt); - break; - } - - first.advance(first_remaining); - cnt = cnt - first_remaining; - self.bufs.pop_front(); - } - } - - fn copy_to_bytes(&mut self, len: usize) -> bytes::Bytes { - match self.bufs.front_mut() { - Some(buf) if len <= buf.remaining() => { - let bytes = buf.copy_to_bytes(len); - if buf.remaining() == 0 { - self.bufs.pop_front(); - } - bytes - } - _ => { - let mut bytes = BytesMut::with_capacity(len); - bytes.put(self.take(len)); - bytes.freeze() - } - } - } -} - -pub enum BytesData { - BufferedBytes(InnerBuffer), - OriginBytes(Bytes), -} - -impl Buf for BytesData { - fn remaining(&self) -> usize { - match self { - BytesData::BufferedBytes(bytes) => bytes.remaining(), - BytesData::OriginBytes(bytes) => bytes.remaining(), - } - } - - fn chunk(&self) -> &[u8] { - match self { - BytesData::BufferedBytes(bytes) => bytes.chunk(), - BytesData::OriginBytes(bytes) => bytes.chunk(), - } - } - - fn advance(&mut self, cnt: usize) { - match self { - BytesData::BufferedBytes(bytes) => bytes.advance(cnt), - BytesData::OriginBytes(bytes) => bytes.advance(cnt), - } - } - - fn copy_to_bytes(&mut self, len: usize) -> bytes::Bytes { - match self { - BytesData::BufferedBytes(bytes) => bytes.copy_to_bytes(len), - BytesData::OriginBytes(bytes) => bytes.copy_to_bytes(len), - } - } -} - -impl Stream for ClonedBytesStream { - type Item = Result; - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let mut_self = self.get_mut(); - - let owned_bytes_stream = mut_self.owned.get_or_insert_with(|| { - let lock = mut_self.shared.lock(); - if let Err(e) = lock { - panic!("bytes streams get shared data lock failed. {}", e); - } - let mut data = lock.unwrap(); - - data.take().expect("cannot get shared bytes streams.") - }); - - if mut_self.replay { - mut_self.replay = false; - if owned_bytes_stream.buf.has_remaining() { - return Poll::Ready(Some(Ok(BytesData::BufferedBytes( - owned_bytes_stream.buf.clone(), - )))); - } - } - - let next = owned_bytes_stream.bytes_stream.poll_next_unpin(cx); - - let next = match ready!(next) { - Some(next) => match next { - Ok(next) => next, - Err(e) => return Poll::Ready(Some(Err(e))), - }, - _ => return Poll::Ready(None), - }; - - let len = next.len(); - owned_bytes_stream.buf.capacity = owned_bytes_stream.buf.capacity.saturating_sub(len); - let next = if owned_bytes_stream.buf.is_capped() { - if owned_bytes_stream.buf.has_remaining() { - owned_bytes_stream.buf.bufs = VecDeque::default(); - } - next - } else { - owned_bytes_stream.buf.push_bytes(next) - }; - - return Poll::Ready(Some(Ok(BytesData::OriginBytes(next)))); - } -} - -#[pin_project] -#[derive(Clone)] -pub struct ClonedBody(#[pin] ClonedBytesStream); - -impl ClonedBody { - pub fn new(inner_body: T) -> Self - where - T: Stream> + Send + 'static, - { - let inner_body = Box::pin(inner_body); - let inner_body = ClonedBytesStream::new(1024 * 64, inner_body); - ClonedBody(inner_body) - } -} - -impl Body for ClonedBody { - type Data = BytesData; - - type Error = Status; - - fn poll_data( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll>> { - self.project().0.poll_next(cx) - } - - fn poll_trailers( - self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll, Self::Error>> { - Poll::Ready(Ok(None)) - } - - fn size_hint(&self) -> http_body::SizeHint { - http_body::SizeHint::default() - } -} - -#[cfg(test)] -pub mod tests { - use super::*; - - #[tokio::test] - async fn test_cloned_bytes_stream() { - let bytes1 = Ok(Bytes::from("hello".as_bytes())); - let bytes2 = Ok(Bytes::from(" world".as_bytes())); - let bytes3 = Ok(Bytes::from(", end test!".as_bytes())); - - let stream = futures_util::stream::iter(vec![bytes1, bytes2, bytes3]); - - let mut origin_stream = ClonedBytesStream::new(64 * 1024, Box::pin(stream)); - let hello_bytes = origin_stream.next().await; - - assert!(hello_bytes.is_some()); - - let hello_bytes = hello_bytes.unwrap(); - - assert!(hello_bytes.is_ok()); - - assert_eq!(bytes_data_to_str(hello_bytes.unwrap()), "hello"); - - let world_bytes = origin_stream.next().await; - - assert!(world_bytes.is_some()); - - let world_bytes = world_bytes.unwrap(); - - assert!(world_bytes.is_ok()); - - assert_eq!(bytes_data_to_str(world_bytes.unwrap()), " world"); - - let end_bytes = origin_stream.next().await; - - assert!(end_bytes.is_some()); - - let end_bytes = end_bytes.unwrap(); - - assert!(end_bytes.is_ok()); - - assert_eq!(bytes_data_to_str(end_bytes.unwrap()), ", end test!"); - - let none_bytes = origin_stream.next().await; - - assert!(none_bytes.is_none()); - } - - #[tokio::test] - async fn test_cloned_bytes_stream_and_replay() { - let bytes1 = Ok(Bytes::from("hello".as_bytes())); - let bytes2 = Ok(Bytes::from(" world".as_bytes())); - let bytes3 = Ok(Bytes::from(", end test!".as_bytes())); - - let stream = futures_util::stream::iter(vec![bytes1, bytes2, bytes3]); - - let mut origin_stream = ClonedBytesStream::new(64 * 1024, Box::pin(stream)); - origin_stream.next().await; - origin_stream.next().await; - origin_stream.next().await; - - let none_bytes = origin_stream.next().await; - assert!(none_bytes.is_none()); - - let mut clone_origin_stream = origin_stream.clone(); - - drop(origin_stream); - - let cached_bytes = clone_origin_stream.next().await; - assert!(cached_bytes.is_some()); - - let cached_bytes = cached_bytes.unwrap(); - - assert!(cached_bytes.is_ok()); - - assert_eq!( - bytes_data_to_str(cached_bytes.unwrap()), - "hello world, end test!" - ); - - let none_bytes = clone_origin_stream.next().await; - assert!(none_bytes.is_none()); - } - - #[tokio::test] - async fn test_replay_stream_continue_poll_next() { - let bytes1 = Ok(Bytes::from("hello".as_bytes())); - let bytes2 = Ok(Bytes::from(" world".as_bytes())); - let bytes3 = Ok(Bytes::from(", end test!".as_bytes())); - - let stream = futures_util::stream::iter(vec![bytes1, bytes2, bytes3]); - - let mut origin_stream = ClonedBytesStream::new(1024 * 64, Box::pin(stream)); - origin_stream.next().await; - origin_stream.next().await; - - let mut clone_origin_stream = origin_stream.clone(); - - drop(origin_stream); - - let cached_bytes = clone_origin_stream.next().await; - assert!(cached_bytes.is_some()); - - let cached_bytes = cached_bytes.unwrap(); - - assert!(cached_bytes.is_ok()); - - assert_eq!(bytes_data_to_str(cached_bytes.unwrap()), "hello world"); - - let next_bytes = clone_origin_stream.next().await; - - assert!(next_bytes.is_some()); - - let next_bytes = next_bytes.unwrap(); - - assert!(next_bytes.is_ok()); - - assert_eq!(bytes_data_to_str(next_bytes.unwrap()), ", end test!"); - - let none_bytes = clone_origin_stream.next().await; - assert!(none_bytes.is_none()); - } - - #[tokio::test] - async fn test_cloned_bytes_stream_reached_max_capacity() { - let bytes1 = Ok(Bytes::from("hello".as_bytes())); - let bytes2 = Ok(Bytes::from(" world".as_bytes())); - let bytes3 = Ok(Bytes::from(", end test!".as_bytes())); - - let stream = futures_util::stream::iter(vec![bytes1, bytes2, bytes3]); - - let mut origin_stream = ClonedBytesStream::new(5, Box::pin(stream)); - let hello_bytes = origin_stream.next().await; - - assert!(hello_bytes.is_some()); - - let hello_bytes = hello_bytes.unwrap(); - - assert!(hello_bytes.is_ok()); - - assert_eq!(bytes_data_to_str(hello_bytes.unwrap()), "hello"); - - let world_bytes = origin_stream.next().await; - - assert!(world_bytes.is_some()); - - let world_bytes = world_bytes.unwrap(); - - assert!(world_bytes.is_ok()); - - assert_eq!(bytes_data_to_str(world_bytes.unwrap()), " world"); - - let mut cloned_origin_stream = origin_stream.clone(); - - drop(origin_stream); - - let end_bytes = cloned_origin_stream.next().await; - - assert!(end_bytes.is_some()); - - let end_bytes = end_bytes.unwrap(); - - assert!(end_bytes.is_ok()); - - assert_eq!(bytes_data_to_str(end_bytes.unwrap()), ", end test!"); - - let none_bytes = cloned_origin_stream.next().await; - assert!(none_bytes.is_none()); - } - - fn bytes_data_to_str(mut bytes_data: BytesData) -> String { - let len = bytes_data.remaining(); - let bytes = bytes_data.copy_to_bytes(len); - String::from_utf8(bytes.to_vec()).unwrap() - } -} diff --git a/dubbo/src/triple/client/triple.rs b/dubbo/src/triple/client/triple.rs index 22273415..b81661cd 100644 --- a/dubbo/src/triple/client/triple.rs +++ b/dubbo/src/triple/client/triple.rs @@ -19,10 +19,10 @@ use std::str::FromStr; use futures_util::{future, stream, StreamExt, TryStreamExt}; +use aws_smithy_http::body::SdkBody; use http::HeaderValue; use super::builder::ClientBuilder; -use super::replay::ClonedBody; use crate::codegen::RpcInvocation; use crate::{ @@ -57,9 +57,8 @@ impl TripleClient { &self, uri: http::Uri, path: http::uri::PathAndQuery, - body: ClonedBody, - invocation: RpcInvocation, - ) -> http::Request { + body: SdkBody, + ) -> http::Request { let mut parts = uri.into_parts(); parts.path_and_query = Some(path); @@ -111,8 +110,6 @@ impl TripleClient { http::HeaderValue::from_static("gzip"), ); - req.extensions_mut().insert(invocation); - // const ( // TripleContentType = "application/grpc+proto" // TripleUserAgent = "grpc-go/1.35.0-dev" @@ -141,23 +138,25 @@ impl TripleClient { M2: Send + Sync + 'static, { let req = req.map(|m| stream::once(future::ready(m))); - let en = encode( + let body_stream = encode( codec.encoder(), req.into_inner().map(Ok), self.send_compression_encoding, ) .into_stream(); - let body = ClonedBody::new(en); + let body = hyper::Body::wrap_stream(body_stream); + let bytes = hyper::body::to_bytes(body).await.unwrap(); + let sdk_body = SdkBody::from(bytes); let mut conn = self .builder .clone() .unwrap() - .build() + .build(invocation.into()) .unwrap(); let http_uri = http::Uri::from_str(&conn.get_url().to_url()).unwrap(); - let req = self.map_request(http_uri.clone(), path, body, invocation); + let req = self.map_request(http_uri.clone(), path, sdk_body); let response = conn .call(req) @@ -211,19 +210,18 @@ impl TripleClient { self.send_compression_encoding, ) .into_stream(); - - - let body = ClonedBody::new(en); + let body = hyper::Body::wrap_stream(en); + let sdk_body = SdkBody::from(body); let mut conn = self .builder .clone() .unwrap() - .build() + .build(invocation.into()) .unwrap(); let http_uri = http::Uri::from_str(&conn.get_url().to_url()).unwrap(); - let req = self.map_request(http_uri.clone(), path, body, invocation); + let req = self.map_request(http_uri.clone(), path, sdk_body); let response = conn .call(req) @@ -261,17 +259,18 @@ impl TripleClient { self.send_compression_encoding, ) .into_stream(); - let body = ClonedBody::new(en); + let body = hyper::Body::wrap_stream(en); + let sdk_body = SdkBody::from(body); let mut conn = self .builder .clone() .unwrap() - .build() + .build(invocation.into()) .unwrap(); let http_uri = http::Uri::from_str(&conn.get_url().to_url()).unwrap(); - let req = self.map_request(http_uri.clone(), path, body, invocation); + let req = self.map_request(http_uri.clone(), path, sdk_body); // let mut conn = Connection::new().with_host(http_uri); let response = conn @@ -326,18 +325,18 @@ impl TripleClient { self.send_compression_encoding, ) .into_stream(); - - let body = ClonedBody::new(en); + let body = hyper::Body::wrap_stream(en); + let sdk_body = SdkBody::from(body); let mut conn = self .builder .clone() .unwrap() - .build() + .build(invocation.into()) .unwrap(); let http_uri = http::Uri::from_str(&conn.get_url().to_url()).unwrap(); - let req = self.map_request(http_uri.clone(), path, body, invocation); + let req = self.map_request(http_uri.clone(), path, sdk_body); let response = conn .call(req) diff --git a/examples/greeter/src/greeter/client.rs b/examples/greeter/src/greeter/client.rs index eed3e522..1d59cdfa 100644 --- a/examples/greeter/src/greeter/client.rs +++ b/examples/greeter/src/greeter/client.rs @@ -35,20 +35,7 @@ async fn main() { dubbo_logger::init(); let mut builder = ClientBuilder::new(); - - if let Ok(zk_servers) = env::var("ZOOKEEPER_SERVERS") { - let zkr = ZookeeperRegistry::new(&zk_servers); - let directory = RegistryDirectory::new(Box::new(zkr)); - builder = builder.with_directory(Box::new(directory)); - } else if let Ok(nacos_url_str) = env::var("NACOS_URL") { - // NACOS_URL=nacos://mse-96efa264-p.nacos-ans.mse.aliyuncs.com - let nacos_url = Url::from_url(&nacos_url_str).unwrap(); - let registry = NacosRegistry::new(nacos_url); - let directory = RegistryDirectory::new(Box::new(registry)); - builder = builder.with_directory(Box::new(directory)); - } else { - builder = builder.with_host("http://127.0.0.1:8888"); - } + builder.with_host("http://127.0.0.1:8888"); let mut cli = GreeterClient::new(builder); diff --git a/registry/nacos/Cargo.toml b/registry/nacos/Cargo.toml index 2437bc0b..798253a6 100644 --- a/registry/nacos/Cargo.toml +++ b/registry/nacos/Cargo.toml @@ -9,7 +9,7 @@ repository = "https://github.com/apache/dubbo-rust.git" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -nacos-sdk = { version = "0.2.3", features = ["naming", "auth-by-http"] } +nacos-sdk = { version = "0.3", features = ["naming", "auth-by-http"] } dubbo.workspace = true serde_json.workspace = true serde = { workspace = true, features = ["derive"] } From 9cf389206ca4a572d1532fb9915bcb0c6c3ffcbc Mon Sep 17 00:00:00 2001 From: onewe Date: Wed, 15 Nov 2023 21:08:42 +0800 Subject: [PATCH 2/2] Enhance: add cache for routers --- common/base/src/lib.rs | 2 - common/base/src/svc.rs | 7 - dubbo/Cargo.toml | 1 + dubbo/src/cluster/clone_body.rs | 50 ++++-- dubbo/src/cluster/failover.rs | 48 +----- dubbo/src/cluster/mod.rs | 43 ++--- dubbo/src/directory/mod.rs | 179 +++++++++----------- dubbo/src/invoker/mod.rs | 75 ++++++++ dubbo/src/lib.rs | 3 + dubbo/src/loadbalancer/mod.rs | 17 +- {common/base => dubbo}/src/param.rs | 0 dubbo/src/protocol/triple/triple_invoker.rs | 11 +- dubbo/src/registry/n_registry.rs | 142 +++++++++++----- dubbo/src/route/mod.rs | 93 ++++++++-- dubbo/src/svc.rs | 88 ++++++++++ dubbo/src/triple/client/builder.rs | 57 +++---- dubbo/src/triple/client/triple.rs | 90 +++++----- 17 files changed, 555 insertions(+), 351 deletions(-) delete mode 100644 common/base/src/svc.rs create mode 100644 dubbo/src/invoker/mod.rs rename {common/base => dubbo}/src/param.rs (100%) create mode 100644 dubbo/src/svc.rs diff --git a/common/base/src/lib.rs b/common/base/src/lib.rs index 8b488501..cb3bc09a 100644 --- a/common/base/src/lib.rs +++ b/common/base/src/lib.rs @@ -21,8 +21,6 @@ pub mod constants; pub mod node; pub mod url; -pub mod param; -pub mod svc; pub use node::Node; pub use url::Url; \ No newline at end of file diff --git a/common/base/src/svc.rs b/common/base/src/svc.rs deleted file mode 100644 index a4b8995a..00000000 --- a/common/base/src/svc.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub trait NewService { - - type Service; - - fn new_service(&self, target: T) -> Self::Service; - -} \ No newline at end of file diff --git a/dubbo/Cargo.toml b/dubbo/Cargo.toml index 402dc171..84874748 100644 --- a/dubbo/Cargo.toml +++ b/dubbo/Cargo.toml @@ -22,6 +22,7 @@ rustls-pemfile = "1.0.0" tokio-rustls="0.23.4" tokio = { version = "1.0", features = [ "rt-multi-thread", "time", "fs", "macros", "net", "signal", "full" ] } tokio-util = "0.7.9" +tokio-stream = "0.1" prost = "0.10.4" async-trait = "0.1.56" tower-layer.workspace = true diff --git a/dubbo/src/cluster/clone_body.rs b/dubbo/src/cluster/clone_body.rs index 37db93ac..1d6b65a8 100644 --- a/dubbo/src/cluster/clone_body.rs +++ b/dubbo/src/cluster/clone_body.rs @@ -59,17 +59,6 @@ impl BufferedBody { } } - pub fn acquire_owned_body(&mut self) -> &mut OwnedBufferedBody { - self.owned.get_or_insert_with(|| { - let lock = self.shared.lock(); - if let Err(e) = lock { - panic!("buffered body get shared data lock failed. {}", e); - } - let mut data = lock.unwrap(); - - data.take().expect("cannot get shared buffered body.") - }) - } } impl Clone for BufferedBody { @@ -81,7 +70,7 @@ impl Clone for BufferedBody { replay_body: true, replay_trailers: true, is_empty: self.is_empty, - size_hint: self.size_hint, + size_hint: self.size_hint.clone(), } } } @@ -110,7 +99,18 @@ where cx: &mut Context<'_>, ) -> Poll>> { let mut_self = self.get_mut(); - let owned_body = mut_self.acquire_owned_body(); + + let owned_body = mut_self.owned.get_or_insert_with(|| { + let lock = mut_self.shared.lock(); + if let Err(e) = lock { + panic!("buffered body get shared data lock failed. {}", e); + } + let mut data = lock.unwrap(); + + data.take().expect("cannot get shared buffered body.") + }); + + if mut_self.replay_body { mut_self.replay_body = false; @@ -165,8 +165,16 @@ where cx: &mut Context<'_>, ) -> Poll, Self::Error>> { let mut_self = self.get_mut(); - let owned_body = mut_self.acquire_owned_body(); + let owned_body = mut_self.owned.get_or_insert_with(|| { + let lock = mut_self.shared.lock(); + if let Err(e) = lock { + panic!("buffered body get shared data lock failed. {}", e); + } + let mut data = lock.unwrap(); + data.take().expect("cannot get shared buffered body.") + }); + if mut_self.replay_trailers { mut_self.replay_trailers = false; if let Some(ref trailers) = owned_body.trailers { @@ -199,7 +207,7 @@ where } fn size_hint(&self) -> http_body::SizeHint { - self.size_hint + self.size_hint.clone() } @@ -320,7 +328,6 @@ impl Buf for BytesData { } #[pin_project] -#[derive(Clone)] pub struct CloneBody(#[pin] BufferedBody); impl CloneBody @@ -362,3 +369,14 @@ where self.0.size_hint() } } + + +impl Clone for CloneBody +where + B: http_body::Body + Unpin, + B::Error: Into, +{ + fn clone(&self) -> Self { + Self(self.0.clone()) + } +} \ No newline at end of file diff --git a/dubbo/src/cluster/failover.rs b/dubbo/src/cluster/failover.rs index 033c2b09..1a8149c6 100644 --- a/dubbo/src/cluster/failover.rs +++ b/dubbo/src/cluster/failover.rs @@ -1,12 +1,11 @@ -use std::{task::Poll, marker::PhantomData}; +use std::task::Poll; -use dubbo_base::{svc::NewService, param::Param}; use futures_util::future; use http::Request; use tower::{ServiceExt, retry::Retry, util::Oneshot}; use tower_service::Service; - -use crate::{codegen::RpcInvocation, StdError}; + +use crate::StdError; pub struct Failover { inner: N // loadbalancer service @@ -14,48 +13,13 @@ pub struct Failover { #[derive(Clone)] pub struct FailoverPolicy; - - -pub struct NewFailover { - inner: N, // new loadbalancer service - _mark: PhantomData -} - - -impl NewFailover { + +impl Failover { pub fn new(inner: N) -> Self { - NewFailover { - inner, - _mark: PhantomData, - } + Self { inner } } -} - -impl NewService for NewFailover -where - T: Param, - // new loadbalancer service - N: NewService, - // loadbalancer service - N::Service: Service + Send + Clone + 'static, - >::Future: Send, - >::Error: Into + Send + Sync -{ - - type Service = Failover; - - fn new_service(&self, target: T) -> Self::Service { - // loadbalancer service - let inner = self.inner.new_service(target); - - Failover { - inner - } - } - } - impl tower::retry::Policy, Res, E> for FailoverPolicy where diff --git a/dubbo/src/cluster/mod.rs b/dubbo/src/cluster/mod.rs index c5e50cfa..47394e2e 100644 --- a/dubbo/src/cluster/mod.rs +++ b/dubbo/src/cluster/mod.rs @@ -15,68 +15,57 @@ * limitations under the License. */ -use std::marker::PhantomData; -use dubbo_base::{svc::NewService, param::Param}; use http::Request; use tower_service::Service; -use crate::{codegen::RpcInvocation, cluster::failover::NewFailover, StdError}; +use crate::{codegen::RpcInvocation, StdError, svc::NewService, param::Param}; use self::{failover::Failover, clone_body::CloneBody}; -mod clone_body; +mod clone_body; mod clone_invoker; mod failover; -pub struct NewClusterService { - inner: S, // new loadbalancer service - _mark: PhantomData +pub struct NewCluster { + inner: N, // new loadbalancer service } -pub struct ClusterService { +pub struct Cluster { inner: S // failover service } -impl NewClusterService -{ +impl NewCluster { - pub fn layer() -> impl tower_layer::Layer { - tower_layer::layer_fn(|inner: S| { - NewClusterService { + pub fn layer() -> impl tower_layer::Layer { + tower_layer::layer_fn(|inner: N| { + NewCluster { inner, // new loadbalancer service - _mark: PhantomData } }) } } -impl NewService for NewClusterService +impl NewService for NewCluster where T: Param, // new loadbalancer service - S: NewService + Clone, - // loadbalancer service - Svc: Service + Clone + Send, - Svc::Error: Into + Send + Sync, - Svc::Future: Send + S: NewService, { - type Service = ClusterService>; - + type Service = Cluster>; + fn new_service(&self, target: T) -> Self::Service { - // Failover - let inner = NewFailover::new(self.inner.clone()).new_service(target); - ClusterService { - inner + Cluster { + inner: Failover::new(self.inner.new_service(target)) } } } -impl Service> for ClusterService +impl Service> for Cluster where S: Service>>, B: http_body::Body + Unpin, diff --git a/dubbo/src/directory/mod.rs b/dubbo/src/directory/mod.rs index 85a463e6..0861d32a 100644 --- a/dubbo/src/directory/mod.rs +++ b/dubbo/src/directory/mod.rs @@ -18,23 +18,27 @@ use core::panic; use std::{ hash::Hash, - pin::Pin, - task::{Context, Poll}, collections::HashMap, sync::{Arc, Mutex}, marker::PhantomData, + task::{Context, Poll}, collections::HashMap, sync::{Arc, Mutex}, }; -use crate::{StdError, codegen::RpcInvocation, invocation::Invocation}; -use dubbo_base::{param::Param, svc::NewService}; -use futures_core::Future; +use crate::{StdError, codegen::RpcInvocation, invocation::Invocation, registry::n_registry::Registry, invoker::NewInvoker, svc::NewService, param::Param}; use futures_util::future::{poll_fn, self}; -use pin_project::pin_project; -use tokio::{sync::{watch, Notify}, select}; +use tokio::{sync::{watch, Notify, mpsc::channel}, select}; +use tokio_stream::wrappers::ReceiverStream; use tower::{ - discover::{Change, Discover}, ServiceExt, buffer::Buffer, util::{FutureService, Oneshot}, + discover::{Change, Discover}, buffer::Buffer, }; use tower_service::Service; -pub struct NewCachedDirectory(PhantomData); +type BufferedDirectory = Buffer, StdError>>>, ()>; + +pub struct NewCachedDirectory +where + N: Registry + Clone + Send + Sync + 'static, +{ + inner: CachedDirectory, RpcInvocation> +} pub struct CachedDirectory @@ -48,20 +52,11 @@ where pub struct NewDirectory { - // service registry + // registry inner: N, } - -#[pin_project] -pub struct NewDirectoryFuture -where - S: Service<()>, - S::Response: Discover -{ - #[pin] - inner: Oneshot -} + #[derive(Clone)] pub struct Directory @@ -72,38 +67,39 @@ where close: Arc } -impl NewCachedDirectory { - pub fn new() -> Self { - NewCachedDirectory(PhantomData) + +impl NewCachedDirectory +where + N: Registry + Clone + Send + Sync + 'static, +{ + + pub fn layer() -> impl tower_layer::Layer { + tower_layer::layer_fn(|inner: N| { + NewCachedDirectory { + // inner is registry + inner: CachedDirectory::new(NewDirectory::new(inner)), + } + }) } } -impl NewService for NewCachedDirectory +impl NewService for NewCachedDirectory where T: Param, // service registry - N: NewService, - N::Service: Service<()> + Send, - // Discover service - >::Response: Discover + Unpin + Send, - >::Error: Into, - >::Future: Unpin + Send, - // Discover::Key - <>::Response as Discover>::Key: Hash + Eq + Clone + Send, - // Discover::Service = new invoker - <>::Response as Discover>::Service: NewService<()> + Clone + Send + Sync, + N: Registry + Clone + Send + Sync + 'static, { - type Service = CachedDirectory, T>; + type Service = BufferedDirectory; - fn new_service(&self, target: N) -> Self::Service { - - CachedDirectory::new(NewDirectory::new(target)) + fn new_service(&self, target: T) -> Self::Service { + + self.inner.new_service(target.param()) } } - + impl CachedDirectory where N: NewService @@ -116,8 +112,8 @@ where } } } - - + + impl NewService for CachedDirectory where T: Param, @@ -152,91 +148,70 @@ impl NewDirectory { inner } } -} +} impl NewService for NewDirectory where T: Param, // service registry - N: NewService, - N::Service: Service<()> + Send, - // Discover service - >::Response: Discover + Unpin + Send, - >::Future: Unpin + Send, - >::Error: Into, - // Discover::Key - <>::Response as Discover>::Key: Hash + Eq + Clone + Send, - // Discover::service = new invoker - <>::Response as Discover>::Service: Send + Sync + Clone + NewService<()>, + N: Registry + Clone + Send + Sync + 'static, { - type Service = Buffer>::Service>, Directory<>::Response>>, ()>; + type Service = BufferedDirectory; fn new_service(&self, target: T) -> Self::Service { - let discovery_service = self.inner.new_service(target); - Buffer::new(FutureService::new(NewDirectoryFuture::new(discovery_service)), 10) - } - -} + + let service_name = target.param().get_target_service_unique_name(); + let registry = self.inner.clone(); + let (tx, rx) = channel(1024); -impl NewDirectoryFuture -where - // Discover service - S: Service<()>, - S::Response: Discover -{ - pub fn new(inner: S) -> Self { - NewDirectoryFuture { - inner: inner.oneshot(()) - } - } -} - + tokio::spawn(async move { + let receiver = registry.subscribe(service_name).await; + match receiver { + Err(e) => { + // error!("discover stream error: {}", e); + + }, + Ok(mut receiver) => { + loop { + let change = receiver.recv().await; + match change { + None => { + // debug!("discover stream closed."); + break; + }, + Some(change) => { + let _ = tx.send(change); + } + } + } + } + } -impl Future for NewDirectoryFuture -where - // Discover service - S: Service<()>, - // Discover - S::Response: Discover + Unpin + Send, - // the key may be dubbo url - ::Key: Hash + Eq + Clone + Send, - // error - S::Error: Into, - // new invoker - ::Service: NewService<()> + Clone + Send + Sync -{ - type Output = Result, StdError>; + }); - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let this = self.project(); + Buffer::new(Directory::new(ReceiverStream::new(rx)), 1024) + } - this.inner.poll(cx).map(|poll_ret| { - poll_ret.map(|discover| { - Directory::new(discover) - }) - }).map_err(|e| { - e.into() - }) +} - } -} impl Directory where // Discover - D: Discover + Unpin + Send, + D: Discover + Unpin + Send + 'static, // the key may be dubbo url D::Key: Hash + Eq + Clone + Send, // invoker new service D::Service: NewService<()> + Clone + Send + Sync, -{ +{ - pub fn new(mut discover: D) -> Self { + pub fn new(discover: D) -> Self { + + let mut discover = Box::pin(discover); - let pin_discover = Pin::new(&mut discover); let (tx, rx) = watch::channel(Vec::new()); let close = Arc::new(Notify::new()); let close_clone = close.clone(); @@ -248,9 +223,9 @@ where let changed = select! { _ = close_clone.notified() => { // info!("discover stream closed.") - return; + return; }, - changed = poll_fn(|cx| pin_discover.poll_discover(cx)) => { + changed = poll_fn(|cx| discover.as_mut().poll_discover(cx)) => { changed } }; diff --git a/dubbo/src/invoker/mod.rs b/dubbo/src/invoker/mod.rs new file mode 100644 index 00000000..81bcb68f --- /dev/null +++ b/dubbo/src/invoker/mod.rs @@ -0,0 +1,75 @@ +use dubbo_base::Url; +use tower_service::Service; + +use crate::{codegen::TripleInvoker, param::Param, svc::NewService}; + +#[derive(Clone)] +pub struct NewInvoker { + url: Url +} + +pub enum InvokerComponent { + TripleInvoker(TripleInvoker) +} + + +impl NewInvoker { + pub fn new(url: Url) -> Self { + Self { + url + } + } +} + +impl From for NewInvoker { + fn from(url: String) -> Self { + Self { + url: Url::from_url(&url).unwrap() + } + } +} + +impl Param for NewInvoker { + fn param(&self) -> Url { + self.url.clone() + } +} + +impl NewService<()> for NewInvoker { + type Service = InvokerComponent; + fn new_service(&self, _: ()) -> Self::Service { + // todo create another invoker + InvokerComponent::TripleInvoker(TripleInvoker::new(self.url.clone())) + } +} + + +impl Service> for InvokerComponent +where + B: http_body::Body + Unpin + Send + 'static, + B::Error: Into, + B::Data: Send + Unpin, +{ + type Response = http::Response; + + type Error = crate::Error; + + type Future = crate::BoxFuture; + + fn call(&mut self, req: http::Request) -> Self::Future { + match self { + InvokerComponent::TripleInvoker(invoker) => invoker.call(req), + } + } + + fn poll_ready( + &mut self, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + match self { + InvokerComponent::TripleInvoker(invoker) => >>::poll_ready(invoker, cx), + } + } +} + +// InvokerComponent::TripleInvoker(invoker) => >>::poll_ready(invoker, cx), \ No newline at end of file diff --git a/dubbo/src/lib.rs b/dubbo/src/lib.rs index c614d333..252e01c7 100644 --- a/dubbo/src/lib.rs +++ b/dubbo/src/lib.rs @@ -29,6 +29,9 @@ pub mod utils; pub mod directory; pub mod route; pub mod loadbalancer; +pub mod invoker; +pub mod param; +pub mod svc; use http_body::Body; use std::{future::Future, pin::Pin}; diff --git a/dubbo/src/loadbalancer/mod.rs b/dubbo/src/loadbalancer/mod.rs index d4feae75..822794cc 100644 --- a/dubbo/src/loadbalancer/mod.rs +++ b/dubbo/src/loadbalancer/mod.rs @@ -1,25 +1,24 @@ -use dubbo_base::{svc::NewService, param::Param}; use futures_core::future::BoxFuture; use tower::{discover::ServiceList, ServiceExt}; use tower_service::Service; -use crate::{codegen::RpcInvocation, StdError}; - +use crate::{codegen::RpcInvocation, StdError, svc::NewService, param::Param}; + pub struct NewLoadBalancer { - inner: N -} + inner: N, +} #[derive(Clone)] pub struct LoadBalancer { inner: S, // Routes service } - impl NewLoadBalancer { pub fn layer() -> impl tower_layer::Layer{ tower_layer::layer_fn(|inner| { + NewLoadBalancer { inner // NewRoutes } @@ -27,11 +26,11 @@ impl NewLoadBalancer { } } -impl NewService for NewLoadBalancer +impl NewService for NewLoadBalancer where T: Param + Clone, // NewRoutes - N: NewService + N: NewService, { type Service = LoadBalancer; @@ -51,7 +50,7 @@ impl Service for LoadBalancer where Req: Send + 'static, // Routes service - N: Service<(), Response = Vec>, + N: Service<(), Response = Vec> + Clone, N::Error: Into + Send, N::Future: Send + 'static, // new invoker diff --git a/common/base/src/param.rs b/dubbo/src/param.rs similarity index 100% rename from common/base/src/param.rs rename to dubbo/src/param.rs diff --git a/dubbo/src/protocol/triple/triple_invoker.rs b/dubbo/src/protocol/triple/triple_invoker.rs index 47867530..685c4344 100644 --- a/dubbo/src/protocol/triple/triple_invoker.rs +++ b/dubbo/src/protocol/triple/triple_invoker.rs @@ -22,7 +22,7 @@ use std::{ }; use tower_service::Service; -use crate::triple::transport::connection::Connection; +use crate::triple::transport::{connection::Connection, self}; #[derive(Clone)] pub struct TripleInvoker { @@ -46,7 +46,12 @@ impl Debug for TripleInvoker { } } -impl Service> for TripleInvoker { +impl Service> for TripleInvoker +where + B: http_body::Body + Unpin + Send + 'static, + B::Error: Into, + B::Data: Send + Unpin, +{ type Response = http::Response; type Error = crate::Error; @@ -61,7 +66,7 @@ impl Service> for TripleInvoker { &mut self, cx: &mut std::task::Context<'_>, ) -> std::task::Poll> { - self.conn.poll_ready(cx) + >>::poll_ready(&mut self.conn, cx) } } diff --git a/dubbo/src/registry/n_registry.rs b/dubbo/src/registry/n_registry.rs index 94d866c1..69300f2a 100644 --- a/dubbo/src/registry/n_registry.rs +++ b/dubbo/src/registry/n_registry.rs @@ -1,75 +1,127 @@ +use std::sync::Arc; + use async_trait::async_trait; -use dubbo_base::{Url, svc::NewService, param::Param}; -use tower::discover::Discover; -use tower_service::Service; +use dubbo_base::Url; +use tokio::sync::mpsc::{Receiver, channel}; +use tower::discover::Change; + + +use crate::{StdError, invoker::NewInvoker}; -use crate::{StdError, codegen::RpcInvocation, invocation::Invocation}; +type DiscoverStream = Receiver, StdError>>; #[async_trait] pub trait Registry { - type Discover: Discover; + async fn register(&self, url: Url) -> Result<(), StdError>; + + async fn unregister(&self, url: Url) -> Result<(), StdError>; + + // todo service_name change to url + async fn subscribe(&self, service_name: String) -> Result; - type Error; + async fn unsubscribe(&self, url: Url) -> Result<(), StdError>; +} - fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> std::task::Poll>; +#[derive(Clone)] +pub struct ArcRegistry { + inner: Arc +} - async fn register(&self, url: Url) -> Result<(), Self::Error>; - - async fn unregister(&self, url: Url) -> Result<(), Self::Error>; - // todo service_name change to url - async fn subscribe(&self, service_name: String) -> Result; +pub enum RegistryComponent { + NacosRegistry, + ZookeeperRegistry, + StaticRegistry(StaticRegistry), +} + - async fn unsubscribe(&self, url: Url) -> Result<(), Self::Error>; +pub struct StaticRegistry { + urls: Vec } +impl ArcRegistry { -pub struct DiscoverService { - inner: N, - service_name: String, -} + pub fn new(registry: impl Registry + Send + Sync + 'static) -> Self { + Self { inner: Arc::new(registry) } + } +} -impl Service<()> for DiscoverService -where - N: Registry, - N::Discover: Discover + Unpin + Send, -{ - type Response = N::Discover; - type Error = StdError; - type Future = std::pin::Pin> + Send>>; +#[async_trait] +impl Registry for ArcRegistry { + + async fn register(&self, url: Url) -> Result<(), StdError> { + self.register(url).await + } - fn poll_ready(&mut self, cx: &mut std::task::Context<'_>) -> std::task::Poll> { - self.inner.poll_ready(cx) + async fn unregister(&self, url: Url) -> Result<(), StdError> { + self.unregister(url).await } - fn call(&mut self, req: ()) -> Self::Future { - let service_name = self.service_name.clone(); - let discover_fut = self.inner.subscribe(service_name); - Box::pin(async move { - discover_fut.await - }) + async fn subscribe(&self, service_name: String) -> Result { + self.subscribe(service_name).await } + async fn unsubscribe(&self, url: Url) -> Result<(), StdError> { + self.unsubscribe(url).await + } } -impl NewService for R -where - T: Param, - R: Registry + Clone, - R::Discover: Discover + Unpin + Send, -{ - type Service = DiscoverService; - fn new_service(&self, target: T) -> Self::Service { - let service_name = target.param().get_target_service_unique_name(); - DiscoverService { - inner: self.clone(), - service_name, +#[async_trait] +impl Registry for RegistryComponent { + async fn register(&self, url: Url) -> Result<(), StdError> { + todo!() + } + + async fn unregister(&self, url: Url) -> Result<(), StdError> { + todo!() + } + + async fn subscribe(&self, service_name: String) -> Result { + todo!() + } + + async fn unsubscribe(&self, url: Url) -> Result<(), StdError> { + todo!() + } +} + + +impl StaticRegistry { + + pub fn new(urls: Vec) -> Self { + Self { + urls } } } +#[async_trait] +impl Registry for StaticRegistry { + async fn register(&self, url: Url) -> Result<(), StdError> { + todo!() + } + + async fn unregister(&self, url: Url) -> Result<(), StdError> { + todo!() + } + + async fn subscribe(&self, service_name: String) -> Result { + let (tx, rx) = channel(self.urls.len()); + for url in self.urls.iter() { + let invoker = NewInvoker::new(url.clone()); + let change = Ok(Change::Insert(service_name.clone(), invoker)); + tx.send(change).await?; + } + + Ok(rx) + } + + async fn unsubscribe(&self, url: Url) -> Result<(), StdError> { + todo!() + } +} \ No newline at end of file diff --git a/dubbo/src/route/mod.rs b/dubbo/src/route/mod.rs index e701846d..dfbe73e3 100644 --- a/dubbo/src/route/mod.rs +++ b/dubbo/src/route/mod.rs @@ -1,15 +1,24 @@ -use dubbo_base::{svc::NewService, param::Param}; +use std::{sync::{Arc, Mutex}, collections::HashMap}; + use futures_core::{Future, ready}; use futures_util::future::Ready; use pin_project::pin_project; use tokio::{sync::watch, pin}; -use tower::util::FutureService; +use tower::{util::FutureService, buffer::Buffer}; use tower_service::Service; -use crate::{StdError, codegen::RpcInvocation}; +use crate::{StdError, codegen::RpcInvocation, svc::NewService, param::Param, invocation::Invocation}; pub struct NewRoutes { inner: N, +} + +pub struct NewRoutesCache +where + N: NewService +{ + inner: N, + cache: Arc>>, } #[pin_project] @@ -26,40 +35,92 @@ pub struct Routes { invokers_receiver: watch::Receiver>, } - impl NewRoutes { - - pub fn layer() -> impl tower_layer::Layer { - tower_layer::layer_fn(|inner: N| { - NewRoutes { - inner, // NewDirectory - } - }) + pub fn new(inner: N) -> Self { + Self { + inner, + } } } impl NewService for NewRoutes where - T: Param + Clone, + T: Param + Clone + Send + 'static, // NewDirectory N: NewService, // Directory - N::Service: Service<(), Response = watch::Receiver>>, + N::Service: Service<(), Response = watch::Receiver>> + Unpin + Send + 'static, + >::Error: Into, // new invoker service - Nsv: NewService<()> + Clone + Nsv: NewService<()> + Clone + Send + Sync + 'static, { - type Service = FutureService, Routes>; + type Service = Buffer>::Service, T>, Routes>, ()>; fn new_service(&self, target: T) -> Self::Service { let inner = self.inner.new_service(target.clone()); - FutureService::new(NewRoutesFuture { + Buffer::new(FutureService::new(NewRoutesFuture { inner, target, + }), 1024) + } +} +impl NewRoutesCache +where + N: NewService, + >::Service: Service<(), Response = watch::Receiver>> + Unpin + Send + 'static, + >::Error: Into, + Nsv: NewService<()> + Clone + Send + Sync + 'static, + +{ + pub fn layer() -> impl tower_layer::Layer>> { + tower_layer::layer_fn(|inner: N| { + NewRoutesCache::new(NewRoutes::new(inner)) }) } + + +} + + +impl NewRoutesCache +where + N: NewService +{ + pub fn new(inner: N) -> Self { + Self { + inner, + cache: Default::default(), + } + } +} + +impl NewService for NewRoutesCache +where + T: Param, + N: NewService, + N::Service: Clone, +{ + type Service = N::Service; + + fn new_service(&self, target: T) -> Self::Service { + let rpc_inv = target.param(); + let service_name = rpc_inv.get_target_service_unique_name(); + + let mut cache = self.cache.lock().expect("RoutesCache get lock failed"); + + let service = cache.get(&service_name); + match service { + Some(service) => service.clone(), + None => { + let service = self.inner.new_service(rpc_inv); + cache.insert(service_name, service.clone()); + service + } + } + } } diff --git a/dubbo/src/svc.rs b/dubbo/src/svc.rs new file mode 100644 index 00000000..56dc3304 --- /dev/null +++ b/dubbo/src/svc.rs @@ -0,0 +1,88 @@ +use std::{sync::Arc, pin::Pin, marker::PhantomData}; + +use futures_core::Future; +use tower::ServiceExt; +use tower_service::Service; + +pub trait NewService { + + type Service; + + fn new_service(&self, target: T) -> Self::Service; + +} + + +pub struct ArcNewService { + inner: Arc + Send + Sync>, +} + + +impl ArcNewService { + pub fn layer() -> impl tower_layer::Layer + Clone + Copy + where + N: NewService + Send + Sync + 'static, + S: Send + 'static, + { + tower_layer::layer_fn(Self::new) + } + + pub fn new(inner: N) -> Self + where + N: NewService + Send + Sync + 'static, + S: Send + 'static, + { + Self { + inner: Arc::new(inner), + } + } +} + +impl Clone for ArcNewService { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + } + } +} + +impl NewService for ArcNewService { + type Service = S; + + fn new_service(&self, t: T) -> S { + self.inner.new_service(t) + } +} + +// inner: Box> + Send>>> + Send>, +pub struct BoxedService { + inner: N, + _mark: PhantomData +} + +impl BoxedService { + + pub fn layer() -> impl tower_layer::Layer{ + tower_layer::layer_fn(|inner: N| { + Self { + inner, + _mark: PhantomData + } + }) + } +} + + +// impl NewService for BoxedService +// where +// N: NewService, +// N::Service: Service + Send, +// >::Future: Send + 'static, +// { +// type Service = Box>::Response, Error = >::Error, Future = Pin>::Response, >::Error>> + Send>>> + Send>; + +// fn new_service(&self, target: T) -> Self::Service { +// let service = self.inner.new_service(target); +// Box::new(service.map_future(|f|Box::pin(f) as _)) +// } +// } diff --git a/dubbo/src/triple/client/builder.rs b/dubbo/src/triple/client/builder.rs index 6f063f42..89d7a00a 100644 --- a/dubbo/src/triple/client/builder.rs +++ b/dubbo/src/triple/client/builder.rs @@ -15,26 +15,28 @@ * limitations under the License. */ + use std::sync::Arc; use crate::{ - cluster::{directory::StaticDirectory, Cluster, Directory, MockCluster, MockDirectory}, - codegen::{RpcInvocation, TripleInvoker}, - protocol::BoxInvoker, - utils::boxed_clone::BoxCloneService, + utils::boxed_clone::BoxCloneService, registry::n_registry::{RegistryComponent, StaticRegistry, ArcRegistry}, route::{NewRoutes, NewRoutesCache}, loadbalancer::NewLoadBalancer, cluster::{NewCluster, Cluster}, directory::NewCachedDirectory, svc::{ArcNewService, NewService, BoxedService}, StdError, codegen::RpcInvocation, BoxBody, }; use aws_smithy_http::body::SdkBody; use dubbo_base::Url; +use tower::ServiceBuilder; pub type ClientBoxService = BoxCloneService, http::Response, crate::Error>; -#[derive(Clone, Debug, Default)] + +pub type ServiceMK = Arc>>>>>; + +#[derive(Default)] pub struct ClientBuilder { pub timeout: Option, pub connector: &'static str, - directory: Option>>, + registry: Option, pub direct: bool, host: String, } @@ -44,7 +46,7 @@ impl ClientBuilder { ClientBuilder { timeout: None, connector: "", - directory: None, + registry: None, direct: false, host: "".to_string(), } @@ -54,9 +56,9 @@ impl ClientBuilder { Self { timeout: None, connector: "", - directory: Some(Arc::new(Box::new(StaticDirectory::new(&host)))), + registry: Some(ArcRegistry::new(RegistryComponent::StaticRegistry(StaticRegistry::new(vec![Url::from_url(host).unwrap()])))), direct: true, - host: host.clone().to_string(), + host: host.to_string(), } } @@ -67,31 +69,23 @@ impl ClientBuilder { } } - /// host: http://0.0.0.0:8888 - pub fn with_directory(self, directory: Box) -> Self { - Self { - directory: Some(Arc::new(directory)), - ..self - } - } - - pub fn with_registry_directory(self, registry: RegistryDirectory) -> Self { - Self { - directory: Some(Arc::new(Box::new(registry))), + pub fn with_registry(self, registry: RegistryComponent) -> Self { + Self { + registry: Some(ArcRegistry::new(registry)), ..self } } pub fn with_host(self, host: &'static str) -> Self { Self { - directory: Some(Arc::new(Box::new(StaticDirectory::new(&host)))), + registry: Some(ArcRegistry::new(RegistryComponent::StaticRegistry(StaticRegistry::new(vec![Url::from_url(host).unwrap()])))), ..self } } pub fn with_connector(self, connector: &'static str) -> Self { Self { - connector: connector, + connector, ..self } } @@ -100,15 +94,18 @@ impl ClientBuilder { Self { direct, ..self } } - pub fn build(self, _invocation: Arc) -> Option { - if self.direct { - return Some(Box::new(TripleInvoker::new( - Url::from_url(&self.host).unwrap(), - ))); - } + pub fn build(mut self) -> ServiceMK { + + + let registry = self.registry.take().expect("registry must not be empty"); - let cluster = MockCluster::default().join(Box::new(MockDirectory::new())); + let mk_service = ServiceBuilder::new() + .layer(NewCluster::layer()) + .layer(NewLoadBalancer::layer()) + .layer(NewRoutesCache::layer()) + .layer(NewCachedDirectory::layer()) + .service(registry); - return Some(cluster); + Arc::new(mk_service) } } diff --git a/dubbo/src/triple/client/triple.rs b/dubbo/src/triple/client/triple.rs index b81661cd..091e0207 100644 --- a/dubbo/src/triple/client/triple.rs +++ b/dubbo/src/triple/client/triple.rs @@ -15,41 +15,43 @@ * limitations under the License. */ -use std::str::FromStr; use futures_util::{future, stream, StreamExt, TryStreamExt}; use aws_smithy_http::body::SdkBody; use http::HeaderValue; +use tower_service::Service; -use super::builder::ClientBuilder; +use super::builder::{ClientBuilder, ServiceMK}; use crate::codegen::RpcInvocation; +use crate::svc::NewService; use crate::{ invocation::{IntoStreamingRequest, Metadata, Request, Response}, triple::{codec::Codec, compression::CompressionEncoding, decode::Decoding, encode::encode}, }; -#[derive(Debug, Default, Clone)] +#[derive(Clone)] pub struct TripleClient { pub(crate) send_compression_encoding: Option, - pub(crate) builder: Option, + pub(crate) mk: ServiceMK, } impl TripleClient { pub fn connect(host: String) -> Self { let builder = ClientBuilder::from_static(&host).with_direct(true); + let mk = builder.build(); TripleClient { send_compression_encoding: Some(CompressionEncoding::Gzip), - builder: Some(builder), + mk, } } pub fn new(builder: ClientBuilder) -> Self { TripleClient { send_compression_encoding: Some(CompressionEncoding::Gzip), - builder: Some(builder), + mk: builder.build(), } } @@ -145,21 +147,18 @@ impl TripleClient { ) .into_stream(); let body = hyper::Body::wrap_stream(body_stream); - let bytes = hyper::body::to_bytes(body).await.unwrap(); - let sdk_body = SdkBody::from(bytes); - - let mut conn = self - .builder - .clone() - .unwrap() - .build(invocation.into()) - .unwrap(); - let http_uri = http::Uri::from_str(&conn.get_url().to_url()).unwrap(); - let req = self.map_request(http_uri.clone(), path, sdk_body); - let response = conn - .call(req) + let mut invoker = self.mk.new_service(invocation); + + + let request = http::Request::builder() + .body(body).unwrap(); + + + + let response = invoker + .call(request) .await .map_err(|err| crate::status::Status::from_error(err.into())); @@ -211,20 +210,16 @@ impl TripleClient { ) .into_stream(); let body = hyper::Body::wrap_stream(en); - let sdk_body = SdkBody::from(body); + + let mut invoker = self.mk.new_service(invocation); - let mut conn = self - .builder - .clone() - .unwrap() - .build(invocation.into()) - .unwrap(); - let http_uri = http::Uri::from_str(&conn.get_url().to_url()).unwrap(); - let req = self.map_request(http_uri.clone(), path, sdk_body); + let request = http::Request::builder() + .body(body).unwrap(); - let response = conn - .call(req) + + let response = invoker + .call(request) .await .map_err(|err| crate::status::Status::from_error(err.into())); @@ -260,21 +255,17 @@ impl TripleClient { ) .into_stream(); let body = hyper::Body::wrap_stream(en); - let sdk_body = SdkBody::from(body); + let mut invoker = self.mk.new_service(invocation); + + + let request = http::Request::builder() + .body(body).unwrap(); - let mut conn = self - .builder - .clone() - .unwrap() - .build(invocation.into()) - .unwrap(); - let http_uri = http::Uri::from_str(&conn.get_url().to_url()).unwrap(); - let req = self.map_request(http_uri.clone(), path, sdk_body); // let mut conn = Connection::new().with_host(http_uri); - let response = conn - .call(req) + let response = invoker + .call(request) .await .map_err(|err| crate::status::Status::from_error(err.into())); @@ -326,20 +317,15 @@ impl TripleClient { ) .into_stream(); let body = hyper::Body::wrap_stream(en); - let sdk_body = SdkBody::from(body); + let mut invoker = self.mk.new_service(invocation); - let mut conn = self - .builder - .clone() - .unwrap() - .build(invocation.into()) - .unwrap(); - let http_uri = http::Uri::from_str(&conn.get_url().to_url()).unwrap(); - let req = self.map_request(http_uri.clone(), path, sdk_body); + let request = http::Request::builder() + .body(body).unwrap(); + - let response = conn - .call(req) + let response = invoker + .call(request) .await .map_err(|err| crate::status::Status::from_error(err.into()));