From 81f29f20d9c3d7a9ffc5f44c6af81c66d2afb960 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Thu, 16 Sep 2021 21:35:29 +0200 Subject: [PATCH 01/14] Write mem main loop --- Cargo.lock | 1 + crates/freenet2-node/Cargo.toml | 1 + .../freenet2-node/src/bin/gateway-deploy.rs | 2 +- crates/freenet2-node/src/conn_manager.rs | 11 +- .../src/conn_manager/in_memory.rs | 13 +- crates/freenet2-node/src/lib.rs | 2 +- crates/freenet2-node/src/message.rs | 73 +- crates/freenet2-node/src/node.rs | 7 +- crates/freenet2-node/src/node/in_memory.rs | 31 +- crates/freenet2-node/src/node/libp2p_impl.rs | 11 +- crates/freenet2-node/src/node/op_state.rs | 35 +- crates/freenet2-node/src/operations.rs | 70 +- .../src/operations/_tmp_ring_proto.rs | 781 ++++++++++++++++++ .../freenet2-node/src/operations/join_ring.rs | 100 +++ crates/freenet2-node/src/ring_proto.rs | 632 +------------- crates/freenet2-node/tests/node_startup.rs | 2 +- 16 files changed, 1024 insertions(+), 748 deletions(-) create mode 100644 crates/freenet2-node/src/operations/_tmp_ring_proto.rs create mode 100644 crates/freenet2-node/src/operations/join_ring.rs diff --git a/Cargo.lock b/Cargo.lock index e57315a09..f31911a71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1708,6 +1708,7 @@ dependencies = [ name = "locutus-node" version = "0.0.1" dependencies = [ + "async-trait", "bincode", "config", "crossbeam", diff --git a/crates/freenet2-node/Cargo.toml b/crates/freenet2-node/Cargo.toml index fe900f83f..31e1c7fd8 100644 --- a/crates/freenet2-node/Cargo.toml +++ b/crates/freenet2-node/Cargo.toml @@ -13,6 +13,7 @@ doc = false [dependencies] # anyhow = "1.0.42" +async-trait = "0.1.51" bincode = "1.3" config = { version = "0.11", features = [ "toml" ] } crossbeam = "0.8.1" diff --git a/crates/freenet2-node/src/bin/gateway-deploy.rs b/crates/freenet2-node/src/bin/gateway-deploy.rs index 0382cdf2e..a01150a78 100644 --- a/crates/freenet2-node/src/bin/gateway-deploy.rs +++ b/crates/freenet2-node/src/bin/gateway-deploy.rs @@ -5,5 +5,5 @@ use locutus_node::*; async fn main() -> Result<(), Box> { let key = Keypair::generate_ed25519(); let mut node = NodeConfig::default().with_key(key).build_libp2p()?; - node.listen_on().map_err(|_| "failed to start".into()) + node.listen_on().await.map_err(|_| "failed to start".into()) } diff --git a/crates/freenet2-node/src/conn_manager.rs b/crates/freenet2-node/src/conn_manager.rs index fc627427e..4b1a2aaea 100644 --- a/crates/freenet2-node/src/conn_manager.rs +++ b/crates/freenet2-node/src/conn_manager.rs @@ -6,7 +6,7 @@ use libp2p::{core::PublicKey, PeerId}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use crate::{ - message::{Message, TransactionTypeId, Transaction}, + message::{Message, Transaction, TransactionTypeId}, ring_proto::Location, StdResult, }; @@ -36,6 +36,15 @@ impl Default for ListenerHandle { } } +#[async_trait::async_trait] +pub(crate) trait ConnectionBridge2 { + async fn recv(&self) -> Result; + + async fn send(&self, msg: Message) -> Result<()>; +} + +/// Manage message carrying in a thread safe & friendly manner. +/// /// Types which impl this trait are responsible for the following responsabilities: /// - establishing reliable connections to other peers, /// including any handshake procedures diff --git a/crates/freenet2-node/src/conn_manager/in_memory.rs b/crates/freenet2-node/src/conn_manager/in_memory.rs index 97cba427f..b156e2a22 100644 --- a/crates/freenet2-node/src/conn_manager/in_memory.rs +++ b/crates/freenet2-node/src/conn_manager/in_memory.rs @@ -12,7 +12,7 @@ use crate::{ ring_proto::Location, }; -use super::Transport; +use super::{ConnError, ConnectionBridge2, Transport}; type InboundListenerFn = Box conn_manager::Result<()> + Send + Sync>; @@ -106,6 +106,17 @@ impl MemoryConnManager { } } +#[async_trait::async_trait] +impl ConnectionBridge2 for MemoryConnManager { + async fn recv(&self) -> Result { + todo!() + } + + async fn send(&self, msg: Message) -> Result<(), ConnError> { + todo!() + } +} + impl ConnectionBridge for MemoryConnManager { type Transport = InMemoryTransport; diff --git a/crates/freenet2-node/src/lib.rs b/crates/freenet2-node/src/lib.rs index 50d9d5828..165206917 100644 --- a/crates/freenet2-node/src/lib.rs +++ b/crates/freenet2-node/src/lib.rs @@ -3,7 +3,7 @@ pub mod conn_manager; mod message; mod node; mod operations; -mod probe_proto; +// mod probe_proto; mod ring_proto; pub use conn_manager::PeerKey; diff --git a/crates/freenet2-node/src/message.rs b/crates/freenet2-node/src/message.rs index 64f72e90a..1f36ac5d4 100644 --- a/crates/freenet2-node/src/message.rs +++ b/crates/freenet2-node/src/message.rs @@ -70,53 +70,64 @@ mod sealed_msg_type { #[repr(u8)] #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy, Serialize, Deserialize)] pub(crate) enum TransactionTypeId { - OpenConnection, + JoinRing, Probe, } impl TransactionTypeId { pub const fn enumeration() -> [Self; 2] { - [Self::OpenConnection, Self::Probe] + [Self::JoinRing, Self::Probe] } } macro_rules! transaction_type_enumeration { - (decl struct { $($type:tt -> $var:tt),+ }) => { - $( transaction_type_enumeration!(@conversion $type -> $var); )+ + (decl struct { $($type:tt),+ }) => { + $( transaction_type_enumeration!(@conversion $type); )+ }; - (@conversion $ty:tt -> $var:tt) => { - impl From<(Transaction, $ty)> for Message { - fn from(oc: (Transaction, $ty)) -> Self { - let (tx_id, oc) = oc; - Self::$ty(tx_id, oc) + (@conversion $ty:tt) => { + impl From<$ty> for Message { + fn from(msg: $ty) -> Self { + Self::$ty(msg) } } impl SealedTxType for $ty { fn tx_type_id() -> TransactionTypeId { - TransactionTypeId::$var + TransactionTypeId::$ty } } }; } transaction_type_enumeration!(decl struct { - JoinRequest -> OpenConnection, - JoinResponse -> OpenConnection, - OpenConnection -> OpenConnection, - ProbeRequest -> Probe, - ProbeResponse -> Probe + JoinRing }); } #[derive(Debug, Serialize, Deserialize, Clone)] -pub(crate) enum Message { - // Ring - JoinRequest(Transaction, JoinRequest), - JoinResponse(Transaction, JoinResponse), - OpenConnection(Transaction, OpenConnection), +pub(crate) enum JoinRing { + Req { id: Transaction, msg: JoinRequest }, + OC { id: Transaction, msg: JoinResponse }, + Resp { id: Transaction, msg: JoinResponse }, +} + +impl JoinRing { + pub fn id(&self) -> &Transaction { + use JoinRing::*; + match self { + Req { id, .. } => id, + OC { id, .. } => id, + Resp { id, .. } => id, + } + } +} +#[derive(Debug, Serialize, Deserialize, Clone)] +pub(crate) enum Message { + JoinRing(JoinRing), + /// Failed a transaction, informing of cancellation. + Canceled(Transaction), // Probe ProbeRequest(Transaction, ProbeRequest), ProbeResponse(Transaction, ProbeResponse), @@ -124,35 +135,21 @@ pub(crate) enum Message { impl Message { fn msg_type_repr(&self) -> &'static str { - use Message::*; - match self { - JoinRequest(_, _) => "JoinRequest", - JoinResponse(_, _) => "JoinResponse", - OpenConnection(_, _) => "OpenConnection", - ProbeRequest(_, _) => "ProbeRequest", - ProbeResponse(_, _) => "ProbeResponse", - } + todo!() } pub fn id(&self) -> &Transaction { use Message::*; match self { - JoinRequest(id, _) => id, - JoinResponse(id, _) => id, - OpenConnection(id, _) => id, + JoinRing(op) => op.id(), ProbeRequest(id, _) => id, ProbeResponse(id, _) => id, + Canceled(_) => todo!(), } } pub fn msg_type(&self) -> TransactionTypeId { - match self { - Self::JoinRequest(_, _) => ::msg_type_id(), - Self::JoinResponse(_, _) => ::msg_type_id(), - Self::OpenConnection(_, _) => ::msg_type_id(), - Self::ProbeRequest(_, _) => ::msg_type_id(), - Self::ProbeResponse(_, _) => ::msg_type_id(), - } + todo!() } } diff --git a/crates/freenet2-node/src/node.rs b/crates/freenet2-node/src/node.rs index 2f234e146..e5978a272 100644 --- a/crates/freenet2-node/src/node.rs +++ b/crates/freenet2-node/src/node.rs @@ -5,6 +5,7 @@ use libp2p::{identity, multiaddr::Protocol, Multiaddr, PeerId}; use crate::config::CONF; use self::{in_memory::InMemory, libp2p_impl::NodeLibP2P}; +pub(crate) use op_state::{OpStateError, OpStateStorage}; mod in_memory; mod libp2p_impl; @@ -13,10 +14,10 @@ mod op_state; pub struct Node(NodeImpl); impl Node { - pub fn listen_on(&mut self) -> Result<(), ()> { + pub async fn listen_on(&mut self) -> Result<(), ()> { match self.0 { - NodeImpl::LibP2P(ref mut node) => node.listen_on(), - NodeImpl::InMemory(ref mut node) => node.listen_on(), + NodeImpl::LibP2P(ref mut node) => node.listen_on().await, + NodeImpl::InMemory(ref mut node) => node.listen_on().await, } } } diff --git a/crates/freenet2-node/src/node/in_memory.rs b/crates/freenet2-node/src/node/in_memory.rs index bacc5bcca..10aee900e 100644 --- a/crates/freenet2-node/src/node/in_memory.rs +++ b/crates/freenet2-node/src/node/in_memory.rs @@ -1,4 +1,9 @@ -use crate::{conn_manager::in_memory::MemoryConnManager, NodeConfig, PeerKey}; +use crate::{ + conn_manager::{in_memory::MemoryConnManager, ConnectionBridge2}, + message::Message, + operations::join_ring, + NodeConfig, PeerKey, +}; use super::op_state::OpStateStorage; @@ -6,7 +11,7 @@ pub(super) struct InMemory { peer: PeerKey, listening: bool, conn_manager: MemoryConnManager, - tx_storage: OpStateStorage, + op_storage: OpStateStorage, } impl InMemory { @@ -22,16 +27,30 @@ impl InMemory { peer, listening: true, conn_manager, - tx_storage: OpStateStorage::new(), + op_storage: OpStateStorage::new(), }) } - pub fn listen_on(&mut self) -> Result<(), ()> { + pub async fn listen_on(&mut self) -> Result<(), ()> { if !self.listening { return Err(()); } - loop {} - Ok(()) + loop { + match self.conn_manager.recv().await { + Ok(msg) => match msg { + Message::JoinRing(join_op) => { + join_ring::join_ring(&mut self.op_storage, &mut self.conn_manager, join_op) + .await + .unwrap(); + } + // old: + Message::ProbeRequest(_, _) => todo!(), + Message::ProbeResponse(_, _) => todo!(), + Message::Canceled(_) => todo!(), + }, + Err(_) => break Err(()), + } + } } } diff --git a/crates/freenet2-node/src/node/libp2p_impl.rs b/crates/freenet2-node/src/node/libp2p_impl.rs index 3073cf4d3..a77053e39 100644 --- a/crates/freenet2-node/src/node/libp2p_impl.rs +++ b/crates/freenet2-node/src/node/libp2p_impl.rs @@ -25,7 +25,7 @@ pub struct NodeLibP2P { } impl NodeLibP2P { - pub(super) fn listen_on(&mut self) -> Result<(), ()> { + pub(super) async fn listen_on(&mut self) -> Result<(), ()> { if let Some(conn) = self.listen_on { let listening_addr = super::multiaddr_from_connection(conn); self.swarm.listen_on(listening_addr).unwrap(); @@ -160,12 +160,9 @@ mod tests { time::Duration, }; - use crate::{ - config::tracing::Logger, - node::{InitPeerNode, Node}, - }; - use super::*; + use crate::{config::tracing::Logger, node::InitPeerNode}; + use libp2p::{futures::StreamExt, swarm::SwarmEvent}; use rand::Rng; @@ -228,7 +225,7 @@ mod tests { .with_port(peer1_port) .with_key(peer1_key); let mut peer1 = NodeLibP2P::build(config).unwrap(); - peer1.listen_on().unwrap(); + peer1.listen_on().await.unwrap(); ping_ev_loop(&mut peer1).await }); diff --git a/crates/freenet2-node/src/node/op_state.rs b/crates/freenet2-node/src/node/op_state.rs index 4a9246913..a14cc652b 100644 --- a/crates/freenet2-node/src/node/op_state.rs +++ b/crates/freenet2-node/src/node/op_state.rs @@ -1,9 +1,9 @@ use crate::{ - message::Transaction, - operations::{AssociatedTxType, JoinRingOp, OpsMap}, + message::{Transaction, TransactionTypeId}, + operations::{join_ring, OpsMap}, }; -pub(super) struct OpStateStorage { +pub(crate) struct OpStateStorage { ops: OpsMap, } @@ -12,24 +12,31 @@ impl OpStateStorage { Self { ops: OpsMap::new() } } - pub fn push_join_ring_op(&mut self, id: Transaction, tx: JoinRingOp) -> Result<(), ()> { - let op_type = ::tx_type_id(); - if !matches!(id.tx_type(), OP_TYPE) { - return Err(()); + pub fn push_join_ring_op( + &mut self, + id: Transaction, + tx: join_ring::JoinRingOp, + ) -> Result<(), OpStateError> { + if !matches!(id.tx_type(), TransactionTypeId::JoinRing) { + return Err(OpStateError::IncorrectTxType( + TransactionTypeId::JoinRing, + id.tx_type(), + )); } self.ops.join_ring.insert(id, tx); Ok(()) } - pub fn pop_join_ring_op(&mut self, id: &Transaction) -> Option { + pub fn pop_join_ring_op(&mut self, id: &Transaction) -> Option { self.ops.join_ring.remove(id) } } -#[cfg(test)] -mod tests { - #[test] - fn example() { - todo!() - } +#[derive(Debug, thiserror::Error)] +pub(crate) enum OpStateError { + #[error("unspected transaction type, trying to get a {0:?} from a {1:?}")] + IncorrectTxType(TransactionTypeId, TransactionTypeId), } + +#[cfg(test)] +mod tests {} diff --git a/crates/freenet2-node/src/operations.rs b/crates/freenet2-node/src/operations.rs index b3878eac5..9bb79b865 100644 --- a/crates/freenet2-node/src/operations.rs +++ b/crates/freenet2-node/src/operations.rs @@ -1,26 +1,33 @@ -//! Keeps track of the join operation state in this machine. -use rust_fsm::*; use serde::{Deserialize, Serialize}; -use crate::message::TransactionTypeId; - -pub(crate) use sealed_op_types::{OperationType, OpsMap}; - -state_machine! { - derive(Debug) - pub(crate) JoinRingOp(Connecting) - - Connecting => { - Connecting => OCReceived [OCReceived], - OCReceived => Connected [Connected], - Connected => Connected [Connected], - }, - OCReceived(Connected) => Connected [Connected], +use crate::{ + conn_manager, + message::{Message, TransactionTypeId}, + node::OpStateError, +}; +use join_ring::JoinRingOp; +pub(crate) use sealed_op_types::OpsMap; + +pub(crate) mod join_ring; + +pub(crate) struct OperationResult { + /// Inhabited if there is a message to return to the other peer. + pub return_msg: Option, + /// None if the operation has been completed. + pub state: Option, } #[derive(Debug, Default)] pub struct ProbeOp; +#[derive(Debug, thiserror::Error)] +pub(crate) enum OpError { + #[error(transparent)] + ConnError(#[from] conn_manager::ConnError), + #[error(transparent)] + OpStateManagerError(#[from] OpStateError), +} + /// Get the transaction type associated to a given operation type. pub(crate) trait AssociatedTxType: sealed_op_types::SealedAssocTxType { fn tx_type_id() -> TransactionTypeId; @@ -75,36 +82,7 @@ mod sealed_op_types { } op_type_enumeration!(decl struct { - join_ring: JoinRingOp -> OpenConnection, + join_ring: JoinRingOp -> JoinRing, probe_peers: ProbeOp -> Probe }); } - -#[cfg(test)] -mod tests { - use super::*; - use rust_fsm::StateMachine; - - #[test] - fn join_ring_transitions() { - let mut join_op_host_1 = StateMachine::::new(); - let res = join_op_host_1 - .consume(&JoinRingOpInput::Connecting) - .unwrap() - .unwrap(); - assert!(matches!(res, JoinRingOpOutput::OCReceived)); - - let mut join_op_host_2 = StateMachine::::new(); - let res = join_op_host_2 - .consume(&JoinRingOpInput::OCReceived) - .unwrap() - .unwrap(); - assert!(matches!(res, JoinRingOpOutput::Connected)); - - let res = join_op_host_1 - .consume(&JoinRingOpInput::Connected) - .unwrap() - .unwrap(); - assert!(matches!(res, JoinRingOpOutput::Connected)); - } -} diff --git a/crates/freenet2-node/src/operations/_tmp_ring_proto.rs b/crates/freenet2-node/src/operations/_tmp_ring_proto.rs new file mode 100644 index 000000000..3236c4398 --- /dev/null +++ b/crates/freenet2-node/src/operations/_tmp_ring_proto.rs @@ -0,0 +1,781 @@ +#![allow(unused)] // FIXME: remove this attr + +use std::{ + collections::{BTreeMap, HashSet}, + convert::TryFrom, + fmt::Display, + hash::Hasher, + sync::Arc, + time::{Duration, Instant}, +}; + +use parking_lot::{Mutex, RwLock}; +use serde::{Deserialize, Serialize}; + +use crate::{ + conn_manager::{self, ConnectionBridge, ListenerHandle, PeerKey, PeerKeyLocation, Transport}, + message::{Message, TransactionType, Transaction}, + ring_proto::messages::{JoinRequest, JoinResponse}, + StdResult, +}; + +type Result = StdResult; + +pub(crate) struct RingProtocol { + pub conn_manager: Arc, + peer_key: PeerKey, + /// A location gets assigned once a node joins the network via a gateway, + /// until then it has no location unless the node is a gateway. + pub location: RwLock>, + gateways: RwLock>, + max_hops_to_live: usize, + rnd_if_htl_above: usize, + pub ring: Ring, +} + +impl RingProtocol +where + T: Transport + 'static, + CM: ConnectionBridge + 'static, +{ + fn new( + conn_manager: CM, + peer_key: PeerKey, + max_hops_to_live: usize, + rnd_if_htl_above: usize, + ) -> Arc { + Arc::new(RingProtocol { + conn_manager: Arc::new(conn_manager), + peer_key, + location: RwLock::new(None), + gateways: RwLock::new(HashSet::new()), + max_hops_to_live, + rnd_if_htl_above, + ring: Ring::new(), + }) + } + + pub fn with_location(self: Arc, loc: Location) -> Arc { + *self.location.write() = Some(loc); + self + } + + fn listen_for_close_conn(&self) { + todo!() + } + + fn listen_for_join_req(self: &Arc) -> ListenerHandle { + let self_cp = self.clone(); + let process_join_req = move |sender: PeerKeyLocation, + msg: Message| + -> conn_manager::Result<()> { + let (tx, join_req) = if let Message::JoinRequest(id, join_req) = msg { + (id, join_req) + } else { + return Err(conn_manager::ConnError::UnexpectedResponseMessage(msg)); + }; + + enum ReqType { + Initial, + Proxy, + } + + let peer_key_loc; + let req_type; + let jr_hpt = match join_req { + messages::JoinRequest::Initial { key, hops_to_live } => { + peer_key_loc = PeerKeyLocation { + peer: key, + location: Some(Location::random()), + }; + req_type = ReqType::Initial; + hops_to_live + } + messages::JoinRequest::Proxy { + joiner, + hops_to_live, + } => { + peer_key_loc = joiner; + req_type = ReqType::Proxy; + hops_to_live + } + }; + log::debug!( + "JoinRequest received by {} with HTL {}", + sender + .location + .ok_or(conn_manager::ConnError::LocationUnknown)?, + jr_hpt + ); + + let your_location = self_cp + .location + .read() + .ok_or(conn_manager::ConnError::LocationUnknown)?; + let accepted_by = if self_cp.ring.should_accept( + &your_location, + &peer_key_loc + .location + .ok_or(conn_manager::ConnError::LocationUnknown)?, + ) { + log::debug!( + "Accepting connections to {:?}, establising connection @ {}", + peer_key_loc, + self_cp.peer_key + ); + self_cp.establish_conn(peer_key_loc, tx); + vec![PeerKeyLocation { + peer: self_cp.peer_key, + location: Some(your_location), + }] + } else { + log::debug!("Not accepting new connection sender {:?}", peer_key_loc); + Vec::new() + }; + + log::debug!( + "Sending JoinResponse to {} accepting {} connections", + sender.peer, + accepted_by.len() + ); + let join_response = match req_type { + ReqType::Initial => Message::from(( + tx, + JoinResponse::Initial { + accepted_by: accepted_by.clone(), + your_location: peer_key_loc + .location + .ok_or(conn_manager::ConnError::LocationUnknown)?, + your_peer_id: peer_key_loc.peer, + }, + )), + ReqType::Proxy => Message::from(( + tx, + JoinResponse::Proxy { + accepted_by: accepted_by.clone(), + }, + )), + }; + self_cp.conn_manager.send(peer_key_loc, tx, join_response)?; + // NOTE: this is in practica a jump to: join_ring.join_response_cb + + if jr_hpt > 0 && !self_cp.ring.connections_by_location.read().is_empty() { + let forward_to = if jr_hpt >= self_cp.rnd_if_htl_above { + log::debug!( + "Randomly selecting peer to forward JoinRequest sender {}", + sender.peer + ); + self_cp.ring.random_peer(|p| p.peer != sender.peer) + } else { + log::debug!( + "Selecting close peer to forward request sender {}", + sender.peer + ); + self_cp + .ring + .connections_by_location + .read() + .get( + &peer_key_loc + .location + .ok_or(conn_manager::ConnError::LocationUnknown)?, + ) + .filter(|it| it.peer != sender.peer) + .copied() + }; + + if let Some(forward_to) = forward_to { + let forwarded = Message::from(( + tx, + JoinRequest::Proxy { + joiner: peer_key_loc, + hops_to_live: jr_hpt.min(self_cp.max_hops_to_live) - 1, + }, + )); + + let forwarded_acceptors = + Arc::new(Mutex::new(accepted_by.into_iter().collect::>())); + + log::debug!( + "Forwarding JoinRequest sender {} to {}", + sender.peer, + forward_to.peer + ); + let self_cp2 = self_cp.clone(); + let register_acceptors = + move |jr_sender: PeerKeyLocation, join_resp| -> conn_manager::Result<()> { + if let Message::JoinResponse(tx, resp) = join_resp { + let new_acceptors = match resp { + JoinResponse::Initial { accepted_by, .. } => accepted_by, + JoinResponse::Proxy { accepted_by, .. } => accepted_by, + }; + let fa = &mut *forwarded_acceptors.lock(); + new_acceptors.iter().for_each(|p| { + if !fa.contains(p) { + fa.insert(*p); + } + }); + let msg = Message::from(( + tx, + JoinResponse::Proxy { + accepted_by: new_acceptors, + }, + )); + self_cp2.conn_manager.send(jr_sender, tx, msg)?; + }; + Ok(()) + }; + self_cp.conn_manager.send_with_callback( + forward_to, + tx, + forwarded, + register_acceptors, + )?; + } + } + Ok(()) + }; + self.conn_manager + .listen(::msg_type_id(), process_join_req) + } + + fn join_ring(self: &Arc) -> Result<()> { + if self.conn_manager.transport().is_open() && self.gateways.read().is_empty() { + match *self.location.read() { + Some(loc) => { + log::info!( + "No gateways to join through, listening for connections at loc: {}", + loc + ); + return Ok(()); + } + None => return Err(RingProtoError::Join), + } + } + + // FIXME: this iteration should be shuffled, must write an extension iterator shuffle items "in place" + // the idea here is to limit the amount of gateways being contacted that's why shuffling is required + for gateway in self.gateways.read().iter() { + log::info!( + "Joining ring via {} at {}", + gateway.peer, + gateway + .location + .ok_or(conn_manager::ConnError::LocationUnknown)? + ); + self.conn_manager.add_connection(*gateway, true); + let tx = Transaction::new(::msg_type_id()); + let join_req = messages::JoinRequest::Initial { + key: self.peer_key, + hops_to_live: self.max_hops_to_live, + }; + log::debug!("Sending {:?} to {}", join_req, gateway.peer); + + let ring_proto = self.clone(); + let join_response_cb = + move |sender: PeerKeyLocation, join_res: Message| -> conn_manager::Result<()> { + let (accepted_by, tx) = if let Message::JoinResponse( + incoming_tx, + messages::JoinResponse::Initial { + accepted_by, + your_location, + .. + }, + ) = join_res + { + log::debug!("JoinResponse received from {}", sender.peer,); + if incoming_tx != tx { + return Err(conn_manager::ConnError::UnexpectedTx(tx, incoming_tx)); + } + let loc = &mut *ring_proto.location.write(); + *loc = Some(your_location); + (accepted_by, incoming_tx) + } else { + return Err(conn_manager::ConnError::UnexpectedResponseMessage(join_res)); + }; + + let self_location = &*ring_proto.location.read(); + let self_location = + &self_location.ok_or(conn_manager::ConnError::LocationUnknown)?; + for new_peer_key in accepted_by { + if ring_proto.ring.should_accept( + self_location, + &new_peer_key + .location + .ok_or(conn_manager::ConnError::LocationUnknown)?, + ) { + log::info!("Establishing connection to {}", new_peer_key.peer); + ring_proto.establish_conn(new_peer_key, tx); + } else { + log::debug!("Not accepting connection to {}", new_peer_key.peer); + } + } + + Ok(()) + }; + log::debug!("Initiating JoinRequest transaction: {}", tx); + let msg: Message = (tx, join_req).into(); + self.conn_manager + .send_with_callback(*gateway, tx, msg, join_response_cb)?; + } + + Ok(()) + } + + fn establish_conn(self: &Arc, new_peer: PeerKeyLocation, tx: Transaction) { + self.conn_manager.add_connection(new_peer, false); + let self_cp = self.clone(); + let state = Arc::new(RwLock::new(messages::OpenConnection::Connecting)); + + let state_cp = state.clone(); + let ack_peer = move |peer: PeerKeyLocation, msg: Message| -> conn_manager::Result<()> { + let (tx, oc) = match msg { + Message::OpenConnection(tx, oc) => (tx, oc), + msg => return Err(conn_manager::ConnError::UnexpectedResponseMessage(msg)), + }; + let mut current_state = state_cp.write(); + current_state.transition(oc); + if !current_state.is_connected() { + let open_conn: Message = (tx, *current_state).into(); + log::debug!("Acknowledging OC"); + self_cp + .conn_manager + .send(peer, *open_conn.id(), open_conn)?; + } else { + log::info!( + "{} connected to {}, adding to ring", + self_cp.peer_key, + new_peer.peer + ); + self_cp.conn_manager.send( + peer, + tx, + Message::from((tx, messages::OpenConnection::Connected)), + )?; + self_cp.ring.connections_by_location.write().insert( + new_peer + .location + .ok_or(conn_manager::ConnError::LocationUnknown)?, + new_peer, + ); + } + Ok(()) + }; + self.conn_manager.listen_to_replies(tx, ack_peer); + + let conn_manager = self.conn_manager.clone(); + tokio::spawn(async move { + let curr_time = Instant::now(); + let mut attempts = 0; + while !state.read().is_connected() && curr_time.elapsed() <= Duration::from_secs(30) { + log::debug!( + "Sending {} to {}, number of messages sent: {}", + *state.read(), + new_peer.peer, + attempts + ); + conn_manager.send(new_peer, tx, Message::OpenConnection(tx, *state.read()))?; + attempts += 1; + tokio::time::sleep(Duration::from_millis(200)).await + } + if curr_time.elapsed() > Duration::from_secs(30) { + log::error!("Timed out trying to connect to {}", new_peer.peer); + Err(conn_manager::ConnError::NegotationFailed) + } else { + conn_manager.remove_listener(tx); + log::info!("Success negotiating connection to {}", new_peer.peer); + Ok(()) + } + }); + } +} + +#[derive(Debug)] +pub(crate) struct Ring { + pub connections_by_location: RwLock>, +} + +impl Ring { + const MIN_CONNECTIONS: usize = 10; + const MAX_CONNECTIONS: usize = 20; + + fn new() -> Self { + Ring { + connections_by_location: RwLock::new(BTreeMap::new()), + } + } + + fn should_accept(&self, my_location: &Location, location: &Location) -> bool { + let cbl = &*self.connections_by_location.read(); + if location == my_location || cbl.contains_key(location) { + false + } else if cbl.len() < Self::MIN_CONNECTIONS { + true + } else if cbl.len() >= Self::MAX_CONNECTIONS { + false + } else { + my_location.distance(location) < self.median_distance_to(my_location) + } + } + + fn median_distance_to(&self, location: &Location) -> Distance { + let mut conn_by_dist = self.connections_by_distance(location); + conn_by_dist.sort_by_key(|(k, _)| *k); + let idx = self.connections_by_location.read().len() / 2; + conn_by_dist[idx].0 + } + + pub fn connections_by_distance(&self, to: &Location) -> Vec<(Distance, PeerKeyLocation)> { + self.connections_by_location + .read() + .iter() + .map(|(key, peer)| (key.distance(to), *peer)) + .collect() + } + + fn random_peer(&self, filter_fn: F) -> Option + where + F: FnMut(&&PeerKeyLocation) -> bool, + { + // FIXME: should be optimized + self.connections_by_location + .read() + .values() + .find(filter_fn) + .copied() + } +} + +/// An abstract location on the 1D ring, represented by a real number on the interal [0, 1] +#[derive(Debug, serde::Serialize, serde::Deserialize, Clone, Copy)] +pub struct Location(f64); + +type Distance = Location; + +impl Location { + /// Returns a new random location. + pub fn random() -> Self { + use rand::prelude::*; + let mut rng = rand::thread_rng(); + Location(rng.gen_range(0.0..=1.0)) + } + + /// Compute the distance between two locations. + pub fn distance(&self, other: &Location) -> Distance { + let d = (self.0 - other.0).abs(); + if d < 0.5 { + Location(d) + } else { + Location(1.0 - d) + } + } +} + +impl Display for Location { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.0.to_string().as_str())?; + Ok(()) + } +} + +impl PartialEq for Location { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } +} + +/// Since we don't allow NaN values in the construction of Location +/// we can safely assume that an equivalence relation holds. +impl Eq for Location {} + +impl Ord for Location { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.partial_cmp(other) + .expect("always should return a cmp value") + } +} + +impl PartialOrd for Location { + fn partial_cmp(&self, other: &Self) -> Option { + self.0.partial_cmp(&other.0) + } +} + +impl std::hash::Hash for Location { + fn hash(&self, state: &mut H) { + let bits = self.0.to_bits(); + state.write_u64(bits); + state.finish(); + } +} + +impl TryFrom for Location { + type Error = (); + + fn try_from(value: f64) -> StdResult { + if !(0.0..=1.0).contains(&value) { + Err(()) + } else { + Ok(Location(value)) + } + } +} + +#[derive(thiserror::Error, Debug)] +pub(crate) enum RingProtoError { + #[error("failed while attempting to join a ring")] + Join, + #[error(transparent)] + ConnError(#[from] conn_manager::ConnError), +} + +pub(crate) mod messages { + use super::*; + + #[derive(Debug, Serialize, Deserialize, Clone)] + pub(crate) enum JoinRequest { + Initial { + key: PeerKey, + hops_to_live: usize, + }, + Proxy { + joiner: PeerKeyLocation, + hops_to_live: usize, + }, + } + + #[derive(Debug, Serialize, Deserialize, Clone)] + pub(crate) enum JoinResponse { + Initial { + accepted_by: Vec, + your_location: Location, + your_peer_id: PeerKey, + }, + Proxy { + accepted_by: Vec, + }, + } + + /// A stateful connection attempt. + #[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)] + pub(crate) enum OpenConnection { + OCReceived, + Connecting, + Connected, + } + + impl OpenConnection { + pub fn is_initiated(&self) -> bool { + matches!(self, OpenConnection::Connecting) + } + + pub fn is_connected(&self) -> bool { + matches!(self, OpenConnection::Connected) + } + + pub(super) fn transition(&mut self, other_host_state: Self) { + match (*self, other_host_state) { + (Self::Connected, _) => {} + (_, Self::Connecting) => *self = Self::OCReceived, + (_, Self::OCReceived | Self::Connected) => *self = Self::Connected, + } + } + } + + impl Display for OpenConnection { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "OpenConnection::{:?}", self) + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use libp2p::identity; + use rand::Rng; + + use super::{messages::OpenConnection, *}; + use crate::{ + config::tracing::Logger, + conn_manager::in_memory::MemoryConnManager, + message::ProbeRequest, + probe_proto::{self, ProbeProtocol}, + }; + + #[test] + fn open_connection_state_transition() { + let mut oc0 = OpenConnection::Connecting; + let oc1 = OpenConnection::Connecting; + oc0.transition(oc1); + assert_eq!(oc0, OpenConnection::OCReceived); + + let mut oc0 = OpenConnection::Connecting; + let oc1 = OpenConnection::OCReceived; + oc0.transition(oc1); + assert!(oc0.is_connected()); + + let mut oc0 = OpenConnection::Connecting; + let oc1 = OpenConnection::Connected; + oc0.transition(oc1); + assert!(oc0.is_connected()); + + let mut oc0 = OpenConnection::Connecting; + let oc1 = OpenConnection::OCReceived; + oc0.transition(oc1); + assert!(oc0.is_connected()); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn node0_to_gateway_conn() -> StdResult<(), Box> { + //! Given a network of one node and one gateway test that both are connected. + Logger::init_logger(); + + let ring_protocols = sim_network_builder(1, 1, 0); + tokio::time::sleep(Duration::from_secs(3)).await; + + assert_eq!( + ring_protocols["node-0"] + .ring_protocol + .ring + .connections_by_location + .read() + .len(), + 1 + ); + + assert_eq!( + ring_protocols["gateway"] + .ring_protocol + .ring + .connections_by_location + .read() + .len(), + 1 + ); + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn all_nodes_should_connect() -> StdResult<(), Box> { + //! Given a network of 1000 peers all nodes should have connections. + Logger::init_logger(); + + let mut sim_nodes = sim_network_builder(10, 10, 7); + tokio::time::sleep(Duration::from_secs(300)).await; + // let _hist: Vec<_> = _ring_distribution(sim_nodes.values()).collect(); + + const NUM_PROBES: usize = 10; + let mut probe_responses = Vec::with_capacity(NUM_PROBES); + for probe_idx in 0..NUM_PROBES { + let target = Location::random(); + let idx: usize = rand::thread_rng().gen_range(0..sim_nodes.len()); + let rnd_node = sim_nodes + .get_mut(&format!("node-{}", idx)) + .ok_or("node not found")?; + let probe_response = ProbeProtocol::probe( + rnd_node.ring_protocol.clone(), + Transaction::new(::msg_type_id()), + ProbeRequest { + hops_to_live: 7, + target, + }, + ) + .await + .expect("failed to get probe response"); + probe_responses.push(probe_response); + } + // probe_proto::utils::plot_probe_responses(probe_responses); + + let any_empties = sim_nodes + .values() + .map(|node| { + node.ring_protocol + .ring + .connections_by_location + .read() + .is_empty() + }) + .any(|is_empty| is_empty); + assert!(!any_empties); + + Ok(()) + } + + struct SimulatedNode { + ring_protocol: Arc>, + probe_protocol: Option, + } + + fn sim_network_builder( + network_size: usize, + ring_max_htl: usize, + rnd_if_htl_above: usize, + // _per_node_delay: usize, + ) -> HashMap { + let mut nodes = HashMap::new(); + + // build gateway node + let keypair = identity::Keypair::generate_ed25519(); + let gw_key = keypair.public().into(); + let loc = Location::random(); + let conn_manager = MemoryConnManager::new(true, gw_key, Some(loc)); + let ring_protocol = RingProtocol::new(conn_manager, gw_key, ring_max_htl, rnd_if_htl_above) + .with_location(loc); + let probe_protocol = Some(ProbeProtocol::new(ring_protocol.clone(), loc)); + ring_protocol.listen_for_join_req(); + nodes.insert( + "gateway".to_owned(), + SimulatedNode { + ring_protocol, + probe_protocol, + }, + ); + + // add other nodes to the simulation + for node_no in 0..network_size { + let label = format!("node-{}", node_no); + let keypair = identity::Keypair::generate_ed25519(); + let peer_key = keypair.public().into(); + let conn_manager = MemoryConnManager::new(false, peer_key, None); + let ring_protocol = + RingProtocol::new(conn_manager, peer_key, ring_max_htl, rnd_if_htl_above); + ring_protocol.gateways.write().insert(PeerKeyLocation { + peer: gw_key, + location: Some(loc), + }); + ring_protocol.join_ring().unwrap(); + + nodes.insert( + label, + SimulatedNode { + ring_protocol, + probe_protocol: None, + }, + ); + } + nodes + } + + /// Builds an histogram of the distribution in the ring of each node relative to each other. + fn _ring_distribution<'a>( + nodes: impl Iterator + 'a, + ) -> impl Iterator + 'a { + // TODO: groupby certain intervals + // e.g. grouping func: (it * 200.0).roundToInt().toDouble() / 200.0 + nodes + .map(|node| { + let node_ring = &node.ring_protocol.ring; + let self_loc = node.ring_protocol.location.read().unwrap(); + node_ring + .connections_by_location + .read() + .keys() + .map(|d| self_loc.distance(d)) + .collect::>() + }) + .flatten() + } +} diff --git a/crates/freenet2-node/src/operations/join_ring.rs b/crates/freenet2-node/src/operations/join_ring.rs new file mode 100644 index 000000000..4444a4c5c --- /dev/null +++ b/crates/freenet2-node/src/operations/join_ring.rs @@ -0,0 +1,100 @@ +use rust_fsm::*; + +use crate::{ + conn_manager::ConnectionBridge2, + message::{JoinRing, Message, Transaction}, + node::OpStateStorage, +}; + +use super::{OpError, OperationResult}; + +state_machine! { + derive(Debug) + pub(crate) JoinRingOp(Connecting) + + Connecting => { + Connecting => OCReceived [OCReceived], + OCReceived => Connected [Connected], + Connected => Connected [Connected], + }, + OCReceived(Connected) => Connected [Connected], +} + +pub(crate) async fn join_ring( + op_storage: &mut OpStateStorage, + conn_manager: &mut CB, + join_op: JoinRing, +) -> Result<(), OpError> +where + CB: ConnectionBridge2, +{ + let id = *join_op.id(); + if let Some(state) = op_storage.pop_join_ring_op(&id) { + // was an existing operation + match update_state(state, join_op) { + Err(tx) => { + log::error!("error while processing {}", tx); + conn_manager.send(Message::Canceled(id)).await?; + } + Ok(OperationResult { + return_msg: Some(msg), + state: Some(updated_state), + }) => { + conn_manager.send(msg).await?; + op_storage.push_join_ring_op(id, updated_state)?; + } + Ok(OperationResult { + return_msg: Some(msg), + state: None, + }) => { + // finished the operation at this node, informing back + conn_manager.send(msg).await?; + } + Ok(OperationResult { + return_msg: None, + state: None, + }) => { + // operation finished_completely + } + _ => unreachable!(), + } + } else { + } + Ok(()) +} + +fn update_state( + current_state: JoinRingOp, + other_host_msg: JoinRing, +) -> Result, Transaction> { + todo!() +} + +#[cfg(test)] +mod tests { + use super::*; + use rust_fsm::StateMachine; + + #[test] + fn join_ring_transitions() { + let mut join_op_host_1 = StateMachine::::new(); + let res = join_op_host_1 + .consume(&JoinRingOpInput::Connecting) + .unwrap() + .unwrap(); + assert!(matches!(res, JoinRingOpOutput::OCReceived)); + + let mut join_op_host_2 = StateMachine::::new(); + let res = join_op_host_2 + .consume(&JoinRingOpInput::OCReceived) + .unwrap() + .unwrap(); + assert!(matches!(res, JoinRingOpOutput::Connected)); + + let res = join_op_host_1 + .consume(&JoinRingOpInput::Connected) + .unwrap() + .unwrap(); + assert!(matches!(res, JoinRingOpOutput::Connected)); + } +} diff --git a/crates/freenet2-node/src/ring_proto.rs b/crates/freenet2-node/src/ring_proto.rs index 3236c4398..672d78923 100644 --- a/crates/freenet2-node/src/ring_proto.rs +++ b/crates/freenet2-node/src/ring_proto.rs @@ -1,451 +1,16 @@ -#![allow(unused)] // FIXME: remove this attr +// #![allow(unused)] // FIXME: remove this attr -use std::{ - collections::{BTreeMap, HashSet}, - convert::TryFrom, - fmt::Display, - hash::Hasher, - sync::Arc, - time::{Duration, Instant}, -}; +use std::{convert::TryFrom, fmt::Display, hash::Hasher}; -use parking_lot::{Mutex, RwLock}; use serde::{Deserialize, Serialize}; use crate::{ - conn_manager::{self, ConnectionBridge, ListenerHandle, PeerKey, PeerKeyLocation, Transport}, - message::{Message, TransactionType, Transaction}, - ring_proto::messages::{JoinRequest, JoinResponse}, + conn_manager::{self, PeerKey, PeerKeyLocation}, StdResult, }; type Result = StdResult; -pub(crate) struct RingProtocol { - pub conn_manager: Arc, - peer_key: PeerKey, - /// A location gets assigned once a node joins the network via a gateway, - /// until then it has no location unless the node is a gateway. - pub location: RwLock>, - gateways: RwLock>, - max_hops_to_live: usize, - rnd_if_htl_above: usize, - pub ring: Ring, -} - -impl RingProtocol -where - T: Transport + 'static, - CM: ConnectionBridge + 'static, -{ - fn new( - conn_manager: CM, - peer_key: PeerKey, - max_hops_to_live: usize, - rnd_if_htl_above: usize, - ) -> Arc { - Arc::new(RingProtocol { - conn_manager: Arc::new(conn_manager), - peer_key, - location: RwLock::new(None), - gateways: RwLock::new(HashSet::new()), - max_hops_to_live, - rnd_if_htl_above, - ring: Ring::new(), - }) - } - - pub fn with_location(self: Arc, loc: Location) -> Arc { - *self.location.write() = Some(loc); - self - } - - fn listen_for_close_conn(&self) { - todo!() - } - - fn listen_for_join_req(self: &Arc) -> ListenerHandle { - let self_cp = self.clone(); - let process_join_req = move |sender: PeerKeyLocation, - msg: Message| - -> conn_manager::Result<()> { - let (tx, join_req) = if let Message::JoinRequest(id, join_req) = msg { - (id, join_req) - } else { - return Err(conn_manager::ConnError::UnexpectedResponseMessage(msg)); - }; - - enum ReqType { - Initial, - Proxy, - } - - let peer_key_loc; - let req_type; - let jr_hpt = match join_req { - messages::JoinRequest::Initial { key, hops_to_live } => { - peer_key_loc = PeerKeyLocation { - peer: key, - location: Some(Location::random()), - }; - req_type = ReqType::Initial; - hops_to_live - } - messages::JoinRequest::Proxy { - joiner, - hops_to_live, - } => { - peer_key_loc = joiner; - req_type = ReqType::Proxy; - hops_to_live - } - }; - log::debug!( - "JoinRequest received by {} with HTL {}", - sender - .location - .ok_or(conn_manager::ConnError::LocationUnknown)?, - jr_hpt - ); - - let your_location = self_cp - .location - .read() - .ok_or(conn_manager::ConnError::LocationUnknown)?; - let accepted_by = if self_cp.ring.should_accept( - &your_location, - &peer_key_loc - .location - .ok_or(conn_manager::ConnError::LocationUnknown)?, - ) { - log::debug!( - "Accepting connections to {:?}, establising connection @ {}", - peer_key_loc, - self_cp.peer_key - ); - self_cp.establish_conn(peer_key_loc, tx); - vec![PeerKeyLocation { - peer: self_cp.peer_key, - location: Some(your_location), - }] - } else { - log::debug!("Not accepting new connection sender {:?}", peer_key_loc); - Vec::new() - }; - - log::debug!( - "Sending JoinResponse to {} accepting {} connections", - sender.peer, - accepted_by.len() - ); - let join_response = match req_type { - ReqType::Initial => Message::from(( - tx, - JoinResponse::Initial { - accepted_by: accepted_by.clone(), - your_location: peer_key_loc - .location - .ok_or(conn_manager::ConnError::LocationUnknown)?, - your_peer_id: peer_key_loc.peer, - }, - )), - ReqType::Proxy => Message::from(( - tx, - JoinResponse::Proxy { - accepted_by: accepted_by.clone(), - }, - )), - }; - self_cp.conn_manager.send(peer_key_loc, tx, join_response)?; - // NOTE: this is in practica a jump to: join_ring.join_response_cb - - if jr_hpt > 0 && !self_cp.ring.connections_by_location.read().is_empty() { - let forward_to = if jr_hpt >= self_cp.rnd_if_htl_above { - log::debug!( - "Randomly selecting peer to forward JoinRequest sender {}", - sender.peer - ); - self_cp.ring.random_peer(|p| p.peer != sender.peer) - } else { - log::debug!( - "Selecting close peer to forward request sender {}", - sender.peer - ); - self_cp - .ring - .connections_by_location - .read() - .get( - &peer_key_loc - .location - .ok_or(conn_manager::ConnError::LocationUnknown)?, - ) - .filter(|it| it.peer != sender.peer) - .copied() - }; - - if let Some(forward_to) = forward_to { - let forwarded = Message::from(( - tx, - JoinRequest::Proxy { - joiner: peer_key_loc, - hops_to_live: jr_hpt.min(self_cp.max_hops_to_live) - 1, - }, - )); - - let forwarded_acceptors = - Arc::new(Mutex::new(accepted_by.into_iter().collect::>())); - - log::debug!( - "Forwarding JoinRequest sender {} to {}", - sender.peer, - forward_to.peer - ); - let self_cp2 = self_cp.clone(); - let register_acceptors = - move |jr_sender: PeerKeyLocation, join_resp| -> conn_manager::Result<()> { - if let Message::JoinResponse(tx, resp) = join_resp { - let new_acceptors = match resp { - JoinResponse::Initial { accepted_by, .. } => accepted_by, - JoinResponse::Proxy { accepted_by, .. } => accepted_by, - }; - let fa = &mut *forwarded_acceptors.lock(); - new_acceptors.iter().for_each(|p| { - if !fa.contains(p) { - fa.insert(*p); - } - }); - let msg = Message::from(( - tx, - JoinResponse::Proxy { - accepted_by: new_acceptors, - }, - )); - self_cp2.conn_manager.send(jr_sender, tx, msg)?; - }; - Ok(()) - }; - self_cp.conn_manager.send_with_callback( - forward_to, - tx, - forwarded, - register_acceptors, - )?; - } - } - Ok(()) - }; - self.conn_manager - .listen(::msg_type_id(), process_join_req) - } - - fn join_ring(self: &Arc) -> Result<()> { - if self.conn_manager.transport().is_open() && self.gateways.read().is_empty() { - match *self.location.read() { - Some(loc) => { - log::info!( - "No gateways to join through, listening for connections at loc: {}", - loc - ); - return Ok(()); - } - None => return Err(RingProtoError::Join), - } - } - - // FIXME: this iteration should be shuffled, must write an extension iterator shuffle items "in place" - // the idea here is to limit the amount of gateways being contacted that's why shuffling is required - for gateway in self.gateways.read().iter() { - log::info!( - "Joining ring via {} at {}", - gateway.peer, - gateway - .location - .ok_or(conn_manager::ConnError::LocationUnknown)? - ); - self.conn_manager.add_connection(*gateway, true); - let tx = Transaction::new(::msg_type_id()); - let join_req = messages::JoinRequest::Initial { - key: self.peer_key, - hops_to_live: self.max_hops_to_live, - }; - log::debug!("Sending {:?} to {}", join_req, gateway.peer); - - let ring_proto = self.clone(); - let join_response_cb = - move |sender: PeerKeyLocation, join_res: Message| -> conn_manager::Result<()> { - let (accepted_by, tx) = if let Message::JoinResponse( - incoming_tx, - messages::JoinResponse::Initial { - accepted_by, - your_location, - .. - }, - ) = join_res - { - log::debug!("JoinResponse received from {}", sender.peer,); - if incoming_tx != tx { - return Err(conn_manager::ConnError::UnexpectedTx(tx, incoming_tx)); - } - let loc = &mut *ring_proto.location.write(); - *loc = Some(your_location); - (accepted_by, incoming_tx) - } else { - return Err(conn_manager::ConnError::UnexpectedResponseMessage(join_res)); - }; - - let self_location = &*ring_proto.location.read(); - let self_location = - &self_location.ok_or(conn_manager::ConnError::LocationUnknown)?; - for new_peer_key in accepted_by { - if ring_proto.ring.should_accept( - self_location, - &new_peer_key - .location - .ok_or(conn_manager::ConnError::LocationUnknown)?, - ) { - log::info!("Establishing connection to {}", new_peer_key.peer); - ring_proto.establish_conn(new_peer_key, tx); - } else { - log::debug!("Not accepting connection to {}", new_peer_key.peer); - } - } - - Ok(()) - }; - log::debug!("Initiating JoinRequest transaction: {}", tx); - let msg: Message = (tx, join_req).into(); - self.conn_manager - .send_with_callback(*gateway, tx, msg, join_response_cb)?; - } - - Ok(()) - } - - fn establish_conn(self: &Arc, new_peer: PeerKeyLocation, tx: Transaction) { - self.conn_manager.add_connection(new_peer, false); - let self_cp = self.clone(); - let state = Arc::new(RwLock::new(messages::OpenConnection::Connecting)); - - let state_cp = state.clone(); - let ack_peer = move |peer: PeerKeyLocation, msg: Message| -> conn_manager::Result<()> { - let (tx, oc) = match msg { - Message::OpenConnection(tx, oc) => (tx, oc), - msg => return Err(conn_manager::ConnError::UnexpectedResponseMessage(msg)), - }; - let mut current_state = state_cp.write(); - current_state.transition(oc); - if !current_state.is_connected() { - let open_conn: Message = (tx, *current_state).into(); - log::debug!("Acknowledging OC"); - self_cp - .conn_manager - .send(peer, *open_conn.id(), open_conn)?; - } else { - log::info!( - "{} connected to {}, adding to ring", - self_cp.peer_key, - new_peer.peer - ); - self_cp.conn_manager.send( - peer, - tx, - Message::from((tx, messages::OpenConnection::Connected)), - )?; - self_cp.ring.connections_by_location.write().insert( - new_peer - .location - .ok_or(conn_manager::ConnError::LocationUnknown)?, - new_peer, - ); - } - Ok(()) - }; - self.conn_manager.listen_to_replies(tx, ack_peer); - - let conn_manager = self.conn_manager.clone(); - tokio::spawn(async move { - let curr_time = Instant::now(); - let mut attempts = 0; - while !state.read().is_connected() && curr_time.elapsed() <= Duration::from_secs(30) { - log::debug!( - "Sending {} to {}, number of messages sent: {}", - *state.read(), - new_peer.peer, - attempts - ); - conn_manager.send(new_peer, tx, Message::OpenConnection(tx, *state.read()))?; - attempts += 1; - tokio::time::sleep(Duration::from_millis(200)).await - } - if curr_time.elapsed() > Duration::from_secs(30) { - log::error!("Timed out trying to connect to {}", new_peer.peer); - Err(conn_manager::ConnError::NegotationFailed) - } else { - conn_manager.remove_listener(tx); - log::info!("Success negotiating connection to {}", new_peer.peer); - Ok(()) - } - }); - } -} - -#[derive(Debug)] -pub(crate) struct Ring { - pub connections_by_location: RwLock>, -} - -impl Ring { - const MIN_CONNECTIONS: usize = 10; - const MAX_CONNECTIONS: usize = 20; - - fn new() -> Self { - Ring { - connections_by_location: RwLock::new(BTreeMap::new()), - } - } - - fn should_accept(&self, my_location: &Location, location: &Location) -> bool { - let cbl = &*self.connections_by_location.read(); - if location == my_location || cbl.contains_key(location) { - false - } else if cbl.len() < Self::MIN_CONNECTIONS { - true - } else if cbl.len() >= Self::MAX_CONNECTIONS { - false - } else { - my_location.distance(location) < self.median_distance_to(my_location) - } - } - - fn median_distance_to(&self, location: &Location) -> Distance { - let mut conn_by_dist = self.connections_by_distance(location); - conn_by_dist.sort_by_key(|(k, _)| *k); - let idx = self.connections_by_location.read().len() / 2; - conn_by_dist[idx].0 - } - - pub fn connections_by_distance(&self, to: &Location) -> Vec<(Distance, PeerKeyLocation)> { - self.connections_by_location - .read() - .iter() - .map(|(key, peer)| (key.distance(to), *peer)) - .collect() - } - - fn random_peer(&self, filter_fn: F) -> Option - where - F: FnMut(&&PeerKeyLocation) -> bool, - { - // FIXME: should be optimized - self.connections_by_location - .read() - .values() - .find(filter_fn) - .copied() - } -} - /// An abstract location on the 1D ring, represented by a real number on the interal [0, 1] #[derive(Debug, serde::Serialize, serde::Deserialize, Clone, Copy)] pub struct Location(f64); @@ -588,194 +153,3 @@ pub(crate) mod messages { } } } - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - - use libp2p::identity; - use rand::Rng; - - use super::{messages::OpenConnection, *}; - use crate::{ - config::tracing::Logger, - conn_manager::in_memory::MemoryConnManager, - message::ProbeRequest, - probe_proto::{self, ProbeProtocol}, - }; - - #[test] - fn open_connection_state_transition() { - let mut oc0 = OpenConnection::Connecting; - let oc1 = OpenConnection::Connecting; - oc0.transition(oc1); - assert_eq!(oc0, OpenConnection::OCReceived); - - let mut oc0 = OpenConnection::Connecting; - let oc1 = OpenConnection::OCReceived; - oc0.transition(oc1); - assert!(oc0.is_connected()); - - let mut oc0 = OpenConnection::Connecting; - let oc1 = OpenConnection::Connected; - oc0.transition(oc1); - assert!(oc0.is_connected()); - - let mut oc0 = OpenConnection::Connecting; - let oc1 = OpenConnection::OCReceived; - oc0.transition(oc1); - assert!(oc0.is_connected()); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn node0_to_gateway_conn() -> StdResult<(), Box> { - //! Given a network of one node and one gateway test that both are connected. - Logger::init_logger(); - - let ring_protocols = sim_network_builder(1, 1, 0); - tokio::time::sleep(Duration::from_secs(3)).await; - - assert_eq!( - ring_protocols["node-0"] - .ring_protocol - .ring - .connections_by_location - .read() - .len(), - 1 - ); - - assert_eq!( - ring_protocols["gateway"] - .ring_protocol - .ring - .connections_by_location - .read() - .len(), - 1 - ); - - Ok(()) - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn all_nodes_should_connect() -> StdResult<(), Box> { - //! Given a network of 1000 peers all nodes should have connections. - Logger::init_logger(); - - let mut sim_nodes = sim_network_builder(10, 10, 7); - tokio::time::sleep(Duration::from_secs(300)).await; - // let _hist: Vec<_> = _ring_distribution(sim_nodes.values()).collect(); - - const NUM_PROBES: usize = 10; - let mut probe_responses = Vec::with_capacity(NUM_PROBES); - for probe_idx in 0..NUM_PROBES { - let target = Location::random(); - let idx: usize = rand::thread_rng().gen_range(0..sim_nodes.len()); - let rnd_node = sim_nodes - .get_mut(&format!("node-{}", idx)) - .ok_or("node not found")?; - let probe_response = ProbeProtocol::probe( - rnd_node.ring_protocol.clone(), - Transaction::new(::msg_type_id()), - ProbeRequest { - hops_to_live: 7, - target, - }, - ) - .await - .expect("failed to get probe response"); - probe_responses.push(probe_response); - } - // probe_proto::utils::plot_probe_responses(probe_responses); - - let any_empties = sim_nodes - .values() - .map(|node| { - node.ring_protocol - .ring - .connections_by_location - .read() - .is_empty() - }) - .any(|is_empty| is_empty); - assert!(!any_empties); - - Ok(()) - } - - struct SimulatedNode { - ring_protocol: Arc>, - probe_protocol: Option, - } - - fn sim_network_builder( - network_size: usize, - ring_max_htl: usize, - rnd_if_htl_above: usize, - // _per_node_delay: usize, - ) -> HashMap { - let mut nodes = HashMap::new(); - - // build gateway node - let keypair = identity::Keypair::generate_ed25519(); - let gw_key = keypair.public().into(); - let loc = Location::random(); - let conn_manager = MemoryConnManager::new(true, gw_key, Some(loc)); - let ring_protocol = RingProtocol::new(conn_manager, gw_key, ring_max_htl, rnd_if_htl_above) - .with_location(loc); - let probe_protocol = Some(ProbeProtocol::new(ring_protocol.clone(), loc)); - ring_protocol.listen_for_join_req(); - nodes.insert( - "gateway".to_owned(), - SimulatedNode { - ring_protocol, - probe_protocol, - }, - ); - - // add other nodes to the simulation - for node_no in 0..network_size { - let label = format!("node-{}", node_no); - let keypair = identity::Keypair::generate_ed25519(); - let peer_key = keypair.public().into(); - let conn_manager = MemoryConnManager::new(false, peer_key, None); - let ring_protocol = - RingProtocol::new(conn_manager, peer_key, ring_max_htl, rnd_if_htl_above); - ring_protocol.gateways.write().insert(PeerKeyLocation { - peer: gw_key, - location: Some(loc), - }); - ring_protocol.join_ring().unwrap(); - - nodes.insert( - label, - SimulatedNode { - ring_protocol, - probe_protocol: None, - }, - ); - } - nodes - } - - /// Builds an histogram of the distribution in the ring of each node relative to each other. - fn _ring_distribution<'a>( - nodes: impl Iterator + 'a, - ) -> impl Iterator + 'a { - // TODO: groupby certain intervals - // e.g. grouping func: (it * 200.0).roundToInt().toDouble() / 200.0 - nodes - .map(|node| { - let node_ring = &node.ring_protocol.ring; - let self_loc = node.ring_protocol.location.read().unwrap(); - node_ring - .connections_by_location - .read() - .keys() - .map(|d| self_loc.distance(d)) - .collect::>() - }) - .flatten() - } -} diff --git a/crates/freenet2-node/tests/node_startup.rs b/crates/freenet2-node/tests/node_startup.rs index be0a90b7a..fdf1ad15a 100644 --- a/crates/freenet2-node/tests/node_startup.rs +++ b/crates/freenet2-node/tests/node_startup.rs @@ -5,5 +5,5 @@ use locutus_node::*; async fn start_node() -> Result<(), Box> { let key = Keypair::generate_ed25519(); let mut node = NodeConfig::default().with_key(key).build_in_memory()?; - node.listen_on().map_err(|_| "failed to start".into()) + node.listen_on().await.map_err(|_| "failed to start".into()) } From 9adf2540371015bd07e1964809250e9a5b614ade Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Fri, 17 Sep 2021 18:30:02 +0200 Subject: [PATCH 02/14] Add back ring code --- crates/freenet2-node/src/conn_manager.rs | 66 +-- .../src/conn_manager/in_memory.rs | 139 +++-- crates/freenet2-node/src/message.rs | 48 +- crates/freenet2-node/src/node/in_memory.rs | 2 +- crates/freenet2-node/src/node/op_state.rs | 4 +- crates/freenet2-node/src/operations.rs | 80 +-- .../src/operations/_tmp_ring_proto.rs | 538 ------------------ .../freenet2-node/src/operations/join_ring.rs | 426 +++++++++++++- crates/freenet2-node/src/ring_proto.rs | 129 ++--- 9 files changed, 578 insertions(+), 854 deletions(-) diff --git a/crates/freenet2-node/src/conn_manager.rs b/crates/freenet2-node/src/conn_manager.rs index 4b1a2aaea..ced683907 100644 --- a/crates/freenet2-node/src/conn_manager.rs +++ b/crates/freenet2-node/src/conn_manager.rs @@ -6,7 +6,7 @@ use libp2p::{core::PublicKey, PeerId}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use crate::{ - message::{Message, Transaction, TransactionTypeId}, + message::{Message, Transaction}, ring_proto::Location, StdResult, }; @@ -37,74 +37,12 @@ impl Default for ListenerHandle { } #[async_trait::async_trait] -pub(crate) trait ConnectionBridge2 { +pub(crate) trait ConnectionBridge { async fn recv(&self) -> Result; async fn send(&self, msg: Message) -> Result<()>; } -/// Manage message carrying in a thread safe & friendly manner. -/// -/// Types which impl this trait are responsible for the following responsabilities: -/// - establishing reliable connections to other peers, -/// including any handshake procedures -/// - keep connections alive or reconnecting to other peers -/// - securely transmitting messages between peers -/// - serializing and deserializing messages -/// -/// The implementing types manage the lower level connection details of the the network, -/// usually working at the transport layer over UDP or TCP and performing NAT traversal -/// to establish connections between peers. -pub(crate) trait ConnectionBridge: Send + Sync { - /// The transport being used to manage networking. - type Transport: Transport; - - /// Register a handler callback for when a connection is removed. - // fn on_remove_conn(&self, func: RemoveConnHandler); - - /// Start listening for incoming connections and process any incoming messages with - /// the provided function. - fn listen(&self, tx_type: TransactionTypeId, listen_fn: F) -> ListenerHandle - where - F: Fn(PeerKeyLocation, Message) -> Result<()> + Send + Sync + 'static; - - /// Listens to inbound replies for a previously broadcasted transaction to the network, - /// if a reply is detected performs a callback. - // FIXME: the fn could take arguments by ref if necessary but due to - // https://github.com/rust-lang/rust/issues/70263 it won't compile - // can workaround by wrapping up the fn to express lifetime constraints, - // consider this, meanwhile passing by value is fine - fn listen_to_replies(&self, tx_id: Transaction, callback: F) - where - F: Fn(PeerKeyLocation, Message) -> Result<()> + Send + Sync + 'static; - - fn transport(&self) -> &Self::Transport; - - /// Initiate a connection with a given peer. At this stage NAT traversal - /// has been succesful and the [`Transport`] has established a connection. - fn add_connection(&self, peer_key: PeerKeyLocation, unsolicited: bool); - - /// Sends a message to a given peer which has already been identified and - /// which has established a connection with this peer, registers a callback action - /// with the manager for when a response is received. - fn send_with_callback( - &self, - to: PeerKeyLocation, - tx_id: Transaction, - msg: Message, - callback: F, - ) -> Result<()> - where - F: Fn(PeerKeyLocation, Message) -> Result<()> + Send + Sync + 'static; - - /// Send a message to a given peer which has already been identified and - /// which has established a connection with this peer. - fn send(&self, to: PeerKeyLocation, tx_id: Transaction, msg: Message) -> Result<()>; - - /// Remove a listener for a given transaction. - fn remove_listener(&self, tx_id: Transaction); -} - /// A protocol used to send and receive data over the network. pub(crate) trait Transport { fn is_open(&self) -> bool; diff --git a/crates/freenet2-node/src/conn_manager/in_memory.rs b/crates/freenet2-node/src/conn_manager/in_memory.rs index b156e2a22..430a375b5 100644 --- a/crates/freenet2-node/src/conn_manager/in_memory.rs +++ b/crates/freenet2-node/src/conn_manager/in_memory.rs @@ -5,6 +5,7 @@ use crossbeam::channel::{self, Receiver, Sender}; use once_cell::sync::OnceCell; use parking_lot::{Mutex, RwLock}; +use super::{ConnError, Transport}; use crate::{ config::tracing::Logger, conn_manager::{self, ConnectionBridge, ListenerHandle, PeerKey, PeerKeyLocation}, @@ -12,8 +13,6 @@ use crate::{ ring_proto::Location, }; -use super::{ConnError, ConnectionBridge2, Transport}; - type InboundListenerFn = Box conn_manager::Result<()> + Send + Sync>; type InboundListenerRegistry = RwLock>; @@ -107,7 +106,7 @@ impl MemoryConnManager { } #[async_trait::async_trait] -impl ConnectionBridge2 for MemoryConnManager { +impl ConnectionBridge for MemoryConnManager { async fn recv(&self) -> Result { todo!() } @@ -117,81 +116,81 @@ impl ConnectionBridge2 for MemoryConnManager { } } -impl ConnectionBridge for MemoryConnManager { - type Transport = InMemoryTransport; +// impl ConnectionBridge for MemoryConnManager { +// type Transport = InMemoryTransport; - fn listen(&self, tx_type: TransactionTypeId, listen_fn: F) -> ListenerHandle - where - F: Fn(PeerKeyLocation, Message) -> conn_manager::Result<()> + Send + Sync + 'static, - { - let tx_ty_listener = &self.inbound_listeners[&tx_type]; - let handle_id = ListenerHandle::new(); - tx_ty_listener - .write() - .insert(handle_id, Box::new(listen_fn)); - handle_id - } +// fn listen(&self, tx_type: TransactionTypeId, listen_fn: F) -> ListenerHandle +// where +// F: Fn(PeerKeyLocation, Message) -> conn_manager::Result<()> + Send + Sync + 'static, +// { +// let tx_ty_listener = &self.inbound_listeners[&tx_type]; +// let handle_id = ListenerHandle::new(); +// tx_ty_listener +// .write() +// .insert(handle_id, Box::new(listen_fn)); +// handle_id +// } - fn listen_to_replies(&self, tx: Transaction, callback: F) - where - F: Fn(PeerKeyLocation, Message) -> conn_manager::Result<()> + Send + Sync + 'static, - { - // optimistically try to acquire a lock - if let Some(mut lock) = self.outbound_listeners.try_write() { - lock.insert(tx, Box::new(callback)); - } else { - // it failed, this is being inserted from an other existing closure holding the lock - // send it to the temporal stack queue for posterior insertion - self.pend_listeners - .send((tx, Box::new(callback))) - .expect("full or disconnected"); - } - } +// fn listen_to_replies(&self, tx: Transaction, callback: F) +// where +// F: Fn(PeerKeyLocation, Message) -> conn_manager::Result<()> + Send + Sync + 'static, +// { +// // optimistically try to acquire a lock +// if let Some(mut lock) = self.outbound_listeners.try_write() { +// lock.insert(tx, Box::new(callback)); +// } else { +// // it failed, this is being inserted from an other existing closure holding the lock +// // send it to the temporal stack queue for posterior insertion +// self.pend_listeners +// .send((tx, Box::new(callback))) +// .expect("full or disconnected"); +// } +// } - fn transport(&self) -> &Self::Transport { - &self.transport - } +// fn transport(&self) -> &Self::Transport { +// &self.transport +// } - fn add_connection(&self, _peer_key: PeerKeyLocation, _unsolicited: bool) {} +// fn add_connection(&self, _peer_key: PeerKeyLocation, _unsolicited: bool) {} - fn send_with_callback( - &self, - to: PeerKeyLocation, - tx: Transaction, - msg: Message, - callback: F, - ) -> conn_manager::Result<()> - where - F: Fn(PeerKeyLocation, Message) -> conn_manager::Result<()> + Send + Sync + 'static, - { - // store listening func - self.outbound_listeners - .write() - .insert(tx, Box::new(callback)); +// fn send_with_callback( +// &self, +// to: PeerKeyLocation, +// tx: Transaction, +// msg: Message, +// callback: F, +// ) -> conn_manager::Result<()> +// where +// F: Fn(PeerKeyLocation, Message) -> conn_manager::Result<()> + Send + Sync + 'static, +// { +// // store listening func +// self.outbound_listeners +// .write() +// .insert(tx, Box::new(callback)); - // send the msg - let serialized = bincode::serialize(&msg)?; - self.transport - .send(to.peer, to.location.unwrap(), serialized); - Ok(()) - } +// // send the msg +// let serialized = bincode::serialize(&msg)?; +// self.transport +// .send(to.peer, to.location.unwrap(), serialized); +// Ok(()) +// } - fn send( - &self, - to: PeerKeyLocation, - _tx: Transaction, - msg: Message, - ) -> conn_manager::Result<()> { - let serialized = bincode::serialize(&msg)?; - self.transport - .send(to.peer, to.location.unwrap(), serialized); - Ok(()) - } +// fn send( +// &self, +// to: PeerKeyLocation, +// _tx: Transaction, +// msg: Message, +// ) -> conn_manager::Result<()> { +// let serialized = bincode::serialize(&msg)?; +// self.transport +// .send(to.peer, to.location.unwrap(), serialized); +// Ok(()) +// } - fn remove_listener(&self, tx: Transaction) { - self.outbound_listeners.write().remove(&tx); - } -} +// fn remove_listener(&self, tx: Transaction) { +// self.outbound_listeners.write().remove(&tx); +// } +// } static NETWORK_WIRES: OnceCell<(Sender, Receiver)> = OnceCell::new(); diff --git a/crates/freenet2-node/src/message.rs b/crates/freenet2-node/src/message.rs index 1f36ac5d4..57e195093 100644 --- a/crates/freenet2-node/src/message.rs +++ b/crates/freenet2-node/src/message.rs @@ -3,7 +3,7 @@ use std::{fmt::Display, time::Duration}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::ring_proto::{messages::*, Location}; +use crate::{operations::join_ring::JoinRingMsg, ring_proto::Location}; pub(crate) use sealed_msg_type::TransactionTypeId; /// An transaction is a unique, universal and efficient identifier for any @@ -81,51 +81,31 @@ mod sealed_msg_type { } macro_rules! transaction_type_enumeration { - (decl struct { $($type:tt),+ }) => { - $( transaction_type_enumeration!(@conversion $type); )+ - }; - - (@conversion $ty:tt) => { - impl From<$ty> for Message { - fn from(msg: $ty) -> Self { - Self::$ty(msg) + (decl struct { $( $var:tt -> $ty:tt),+ }) => { + $( + impl From<$ty> for Message { + fn from(msg: $ty) -> Self { + Self::$var(msg) + } } - } - impl SealedTxType for $ty { - fn tx_type_id() -> TransactionTypeId { - TransactionTypeId::$ty + impl SealedTxType for $ty { + fn tx_type_id() -> TransactionTypeId { + TransactionTypeId::$var + } } - } + )+ }; } transaction_type_enumeration!(decl struct { - JoinRing + JoinRing -> JoinRingMsg }); } -#[derive(Debug, Serialize, Deserialize, Clone)] -pub(crate) enum JoinRing { - Req { id: Transaction, msg: JoinRequest }, - OC { id: Transaction, msg: JoinResponse }, - Resp { id: Transaction, msg: JoinResponse }, -} - -impl JoinRing { - pub fn id(&self) -> &Transaction { - use JoinRing::*; - match self { - Req { id, .. } => id, - OC { id, .. } => id, - Resp { id, .. } => id, - } - } -} - #[derive(Debug, Serialize, Deserialize, Clone)] pub(crate) enum Message { - JoinRing(JoinRing), + JoinRing(JoinRingMsg), /// Failed a transaction, informing of cancellation. Canceled(Transaction), // Probe diff --git a/crates/freenet2-node/src/node/in_memory.rs b/crates/freenet2-node/src/node/in_memory.rs index 10aee900e..07cd3f750 100644 --- a/crates/freenet2-node/src/node/in_memory.rs +++ b/crates/freenet2-node/src/node/in_memory.rs @@ -1,5 +1,5 @@ use crate::{ - conn_manager::{in_memory::MemoryConnManager, ConnectionBridge2}, + conn_manager::{in_memory::MemoryConnManager, ConnectionBridge}, message::Message, operations::join_ring, NodeConfig, PeerKey, diff --git a/crates/freenet2-node/src/node/op_state.rs b/crates/freenet2-node/src/node/op_state.rs index a14cc652b..088d3cc7d 100644 --- a/crates/freenet2-node/src/node/op_state.rs +++ b/crates/freenet2-node/src/node/op_state.rs @@ -15,7 +15,7 @@ impl OpStateStorage { pub fn push_join_ring_op( &mut self, id: Transaction, - tx: join_ring::JoinRingOp, + tx: join_ring::JoinRingOpState, ) -> Result<(), OpStateError> { if !matches!(id.tx_type(), TransactionTypeId::JoinRing) { return Err(OpStateError::IncorrectTxType( @@ -27,7 +27,7 @@ impl OpStateStorage { Ok(()) } - pub fn pop_join_ring_op(&mut self, id: &Transaction) -> Option { + pub fn pop_join_ring_op(&mut self, id: &Transaction) -> Option { self.ops.join_ring.remove(id) } } diff --git a/crates/freenet2-node/src/operations.rs b/crates/freenet2-node/src/operations.rs index 9bb79b865..29b89bce1 100644 --- a/crates/freenet2-node/src/operations.rs +++ b/crates/freenet2-node/src/operations.rs @@ -2,11 +2,11 @@ use serde::{Deserialize, Serialize}; use crate::{ conn_manager, - message::{Message, TransactionTypeId}, + message::{Message, Transaction}, node::OpStateError, }; -use join_ring::JoinRingOp; -pub(crate) use sealed_op_types::OpsMap; + +use self::join_ring::JoinRingOpState; pub(crate) mod join_ring; @@ -28,61 +28,31 @@ pub(crate) enum OpError { OpStateManagerError(#[from] OpStateError), } -/// Get the transaction type associated to a given operation type. -pub(crate) trait AssociatedTxType: sealed_op_types::SealedAssocTxType { - fn tx_type_id() -> TransactionTypeId; -} - -impl AssociatedTxType for T -where - T: sealed_op_types::SealedAssocTxType, -{ - fn tx_type_id() -> TransactionTypeId { - ::tx_type_id() - } -} - -mod sealed_op_types { - use super::*; - use crate::message::Transaction; - - pub(crate) trait SealedAssocTxType { - fn tx_type_id() -> TransactionTypeId; - } +macro_rules! op_type_enumeration { + (decl struct { $($field:ident: $var:tt),+ } ) => { + #[repr(u8)] + #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy, Serialize, Deserialize)] + pub(crate) enum OperationType { + $($var,)+ + } - macro_rules! op_type_enumeration { - (decl struct { $($field:ident: $var:tt -> $tx_ty:tt),+ } ) => { - #[repr(u8)] - #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy, Serialize, Deserialize)] - pub(crate) enum OperationType { - $($var,)+ - } - - $( - impl SealedAssocTxType for $var { - fn tx_type_id() -> TransactionTypeId { - TransactionTypeId::$tx_ty - } - } - )+ - #[derive(Debug)] - pub(crate) struct OpsMap { - $( pub $field: std::collections::HashMap),+, - } + #[derive(Debug)] + pub(crate) struct OpsMap { + $( pub $field: std::collections::HashMap),+, + } - impl OpsMap { - pub fn new() -> Self { - Self { - $( $field: std::collections::HashMap::new()),+, - } + impl OpsMap { + pub fn new() -> Self { + Self { + $( $field: std::collections::HashMap::new()),+, } } - }; - } - - op_type_enumeration!(decl struct { - join_ring: JoinRingOp -> JoinRing, - probe_peers: ProbeOp -> Probe - }); + } + }; } + +op_type_enumeration!(decl struct { + join_ring: JoinRingOpState, + probe_peers: ProbeOp +}); diff --git a/crates/freenet2-node/src/operations/_tmp_ring_proto.rs b/crates/freenet2-node/src/operations/_tmp_ring_proto.rs index 3236c4398..755655e79 100644 --- a/crates/freenet2-node/src/operations/_tmp_ring_proto.rs +++ b/crates/freenet2-node/src/operations/_tmp_ring_proto.rs @@ -64,461 +64,6 @@ where todo!() } - fn listen_for_join_req(self: &Arc) -> ListenerHandle { - let self_cp = self.clone(); - let process_join_req = move |sender: PeerKeyLocation, - msg: Message| - -> conn_manager::Result<()> { - let (tx, join_req) = if let Message::JoinRequest(id, join_req) = msg { - (id, join_req) - } else { - return Err(conn_manager::ConnError::UnexpectedResponseMessage(msg)); - }; - - enum ReqType { - Initial, - Proxy, - } - - let peer_key_loc; - let req_type; - let jr_hpt = match join_req { - messages::JoinRequest::Initial { key, hops_to_live } => { - peer_key_loc = PeerKeyLocation { - peer: key, - location: Some(Location::random()), - }; - req_type = ReqType::Initial; - hops_to_live - } - messages::JoinRequest::Proxy { - joiner, - hops_to_live, - } => { - peer_key_loc = joiner; - req_type = ReqType::Proxy; - hops_to_live - } - }; - log::debug!( - "JoinRequest received by {} with HTL {}", - sender - .location - .ok_or(conn_manager::ConnError::LocationUnknown)?, - jr_hpt - ); - - let your_location = self_cp - .location - .read() - .ok_or(conn_manager::ConnError::LocationUnknown)?; - let accepted_by = if self_cp.ring.should_accept( - &your_location, - &peer_key_loc - .location - .ok_or(conn_manager::ConnError::LocationUnknown)?, - ) { - log::debug!( - "Accepting connections to {:?}, establising connection @ {}", - peer_key_loc, - self_cp.peer_key - ); - self_cp.establish_conn(peer_key_loc, tx); - vec![PeerKeyLocation { - peer: self_cp.peer_key, - location: Some(your_location), - }] - } else { - log::debug!("Not accepting new connection sender {:?}", peer_key_loc); - Vec::new() - }; - - log::debug!( - "Sending JoinResponse to {} accepting {} connections", - sender.peer, - accepted_by.len() - ); - let join_response = match req_type { - ReqType::Initial => Message::from(( - tx, - JoinResponse::Initial { - accepted_by: accepted_by.clone(), - your_location: peer_key_loc - .location - .ok_or(conn_manager::ConnError::LocationUnknown)?, - your_peer_id: peer_key_loc.peer, - }, - )), - ReqType::Proxy => Message::from(( - tx, - JoinResponse::Proxy { - accepted_by: accepted_by.clone(), - }, - )), - }; - self_cp.conn_manager.send(peer_key_loc, tx, join_response)?; - // NOTE: this is in practica a jump to: join_ring.join_response_cb - - if jr_hpt > 0 && !self_cp.ring.connections_by_location.read().is_empty() { - let forward_to = if jr_hpt >= self_cp.rnd_if_htl_above { - log::debug!( - "Randomly selecting peer to forward JoinRequest sender {}", - sender.peer - ); - self_cp.ring.random_peer(|p| p.peer != sender.peer) - } else { - log::debug!( - "Selecting close peer to forward request sender {}", - sender.peer - ); - self_cp - .ring - .connections_by_location - .read() - .get( - &peer_key_loc - .location - .ok_or(conn_manager::ConnError::LocationUnknown)?, - ) - .filter(|it| it.peer != sender.peer) - .copied() - }; - - if let Some(forward_to) = forward_to { - let forwarded = Message::from(( - tx, - JoinRequest::Proxy { - joiner: peer_key_loc, - hops_to_live: jr_hpt.min(self_cp.max_hops_to_live) - 1, - }, - )); - - let forwarded_acceptors = - Arc::new(Mutex::new(accepted_by.into_iter().collect::>())); - - log::debug!( - "Forwarding JoinRequest sender {} to {}", - sender.peer, - forward_to.peer - ); - let self_cp2 = self_cp.clone(); - let register_acceptors = - move |jr_sender: PeerKeyLocation, join_resp| -> conn_manager::Result<()> { - if let Message::JoinResponse(tx, resp) = join_resp { - let new_acceptors = match resp { - JoinResponse::Initial { accepted_by, .. } => accepted_by, - JoinResponse::Proxy { accepted_by, .. } => accepted_by, - }; - let fa = &mut *forwarded_acceptors.lock(); - new_acceptors.iter().for_each(|p| { - if !fa.contains(p) { - fa.insert(*p); - } - }); - let msg = Message::from(( - tx, - JoinResponse::Proxy { - accepted_by: new_acceptors, - }, - )); - self_cp2.conn_manager.send(jr_sender, tx, msg)?; - }; - Ok(()) - }; - self_cp.conn_manager.send_with_callback( - forward_to, - tx, - forwarded, - register_acceptors, - )?; - } - } - Ok(()) - }; - self.conn_manager - .listen(::msg_type_id(), process_join_req) - } - - fn join_ring(self: &Arc) -> Result<()> { - if self.conn_manager.transport().is_open() && self.gateways.read().is_empty() { - match *self.location.read() { - Some(loc) => { - log::info!( - "No gateways to join through, listening for connections at loc: {}", - loc - ); - return Ok(()); - } - None => return Err(RingProtoError::Join), - } - } - - // FIXME: this iteration should be shuffled, must write an extension iterator shuffle items "in place" - // the idea here is to limit the amount of gateways being contacted that's why shuffling is required - for gateway in self.gateways.read().iter() { - log::info!( - "Joining ring via {} at {}", - gateway.peer, - gateway - .location - .ok_or(conn_manager::ConnError::LocationUnknown)? - ); - self.conn_manager.add_connection(*gateway, true); - let tx = Transaction::new(::msg_type_id()); - let join_req = messages::JoinRequest::Initial { - key: self.peer_key, - hops_to_live: self.max_hops_to_live, - }; - log::debug!("Sending {:?} to {}", join_req, gateway.peer); - - let ring_proto = self.clone(); - let join_response_cb = - move |sender: PeerKeyLocation, join_res: Message| -> conn_manager::Result<()> { - let (accepted_by, tx) = if let Message::JoinResponse( - incoming_tx, - messages::JoinResponse::Initial { - accepted_by, - your_location, - .. - }, - ) = join_res - { - log::debug!("JoinResponse received from {}", sender.peer,); - if incoming_tx != tx { - return Err(conn_manager::ConnError::UnexpectedTx(tx, incoming_tx)); - } - let loc = &mut *ring_proto.location.write(); - *loc = Some(your_location); - (accepted_by, incoming_tx) - } else { - return Err(conn_manager::ConnError::UnexpectedResponseMessage(join_res)); - }; - - let self_location = &*ring_proto.location.read(); - let self_location = - &self_location.ok_or(conn_manager::ConnError::LocationUnknown)?; - for new_peer_key in accepted_by { - if ring_proto.ring.should_accept( - self_location, - &new_peer_key - .location - .ok_or(conn_manager::ConnError::LocationUnknown)?, - ) { - log::info!("Establishing connection to {}", new_peer_key.peer); - ring_proto.establish_conn(new_peer_key, tx); - } else { - log::debug!("Not accepting connection to {}", new_peer_key.peer); - } - } - - Ok(()) - }; - log::debug!("Initiating JoinRequest transaction: {}", tx); - let msg: Message = (tx, join_req).into(); - self.conn_manager - .send_with_callback(*gateway, tx, msg, join_response_cb)?; - } - - Ok(()) - } - - fn establish_conn(self: &Arc, new_peer: PeerKeyLocation, tx: Transaction) { - self.conn_manager.add_connection(new_peer, false); - let self_cp = self.clone(); - let state = Arc::new(RwLock::new(messages::OpenConnection::Connecting)); - - let state_cp = state.clone(); - let ack_peer = move |peer: PeerKeyLocation, msg: Message| -> conn_manager::Result<()> { - let (tx, oc) = match msg { - Message::OpenConnection(tx, oc) => (tx, oc), - msg => return Err(conn_manager::ConnError::UnexpectedResponseMessage(msg)), - }; - let mut current_state = state_cp.write(); - current_state.transition(oc); - if !current_state.is_connected() { - let open_conn: Message = (tx, *current_state).into(); - log::debug!("Acknowledging OC"); - self_cp - .conn_manager - .send(peer, *open_conn.id(), open_conn)?; - } else { - log::info!( - "{} connected to {}, adding to ring", - self_cp.peer_key, - new_peer.peer - ); - self_cp.conn_manager.send( - peer, - tx, - Message::from((tx, messages::OpenConnection::Connected)), - )?; - self_cp.ring.connections_by_location.write().insert( - new_peer - .location - .ok_or(conn_manager::ConnError::LocationUnknown)?, - new_peer, - ); - } - Ok(()) - }; - self.conn_manager.listen_to_replies(tx, ack_peer); - - let conn_manager = self.conn_manager.clone(); - tokio::spawn(async move { - let curr_time = Instant::now(); - let mut attempts = 0; - while !state.read().is_connected() && curr_time.elapsed() <= Duration::from_secs(30) { - log::debug!( - "Sending {} to {}, number of messages sent: {}", - *state.read(), - new_peer.peer, - attempts - ); - conn_manager.send(new_peer, tx, Message::OpenConnection(tx, *state.read()))?; - attempts += 1; - tokio::time::sleep(Duration::from_millis(200)).await - } - if curr_time.elapsed() > Duration::from_secs(30) { - log::error!("Timed out trying to connect to {}", new_peer.peer); - Err(conn_manager::ConnError::NegotationFailed) - } else { - conn_manager.remove_listener(tx); - log::info!("Success negotiating connection to {}", new_peer.peer); - Ok(()) - } - }); - } -} - -#[derive(Debug)] -pub(crate) struct Ring { - pub connections_by_location: RwLock>, -} - -impl Ring { - const MIN_CONNECTIONS: usize = 10; - const MAX_CONNECTIONS: usize = 20; - - fn new() -> Self { - Ring { - connections_by_location: RwLock::new(BTreeMap::new()), - } - } - - fn should_accept(&self, my_location: &Location, location: &Location) -> bool { - let cbl = &*self.connections_by_location.read(); - if location == my_location || cbl.contains_key(location) { - false - } else if cbl.len() < Self::MIN_CONNECTIONS { - true - } else if cbl.len() >= Self::MAX_CONNECTIONS { - false - } else { - my_location.distance(location) < self.median_distance_to(my_location) - } - } - - fn median_distance_to(&self, location: &Location) -> Distance { - let mut conn_by_dist = self.connections_by_distance(location); - conn_by_dist.sort_by_key(|(k, _)| *k); - let idx = self.connections_by_location.read().len() / 2; - conn_by_dist[idx].0 - } - - pub fn connections_by_distance(&self, to: &Location) -> Vec<(Distance, PeerKeyLocation)> { - self.connections_by_location - .read() - .iter() - .map(|(key, peer)| (key.distance(to), *peer)) - .collect() - } - - fn random_peer(&self, filter_fn: F) -> Option - where - F: FnMut(&&PeerKeyLocation) -> bool, - { - // FIXME: should be optimized - self.connections_by_location - .read() - .values() - .find(filter_fn) - .copied() - } -} - -/// An abstract location on the 1D ring, represented by a real number on the interal [0, 1] -#[derive(Debug, serde::Serialize, serde::Deserialize, Clone, Copy)] -pub struct Location(f64); - -type Distance = Location; - -impl Location { - /// Returns a new random location. - pub fn random() -> Self { - use rand::prelude::*; - let mut rng = rand::thread_rng(); - Location(rng.gen_range(0.0..=1.0)) - } - - /// Compute the distance between two locations. - pub fn distance(&self, other: &Location) -> Distance { - let d = (self.0 - other.0).abs(); - if d < 0.5 { - Location(d) - } else { - Location(1.0 - d) - } - } -} - -impl Display for Location { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.0.to_string().as_str())?; - Ok(()) - } -} - -impl PartialEq for Location { - fn eq(&self, other: &Self) -> bool { - self.0 == other.0 - } -} - -/// Since we don't allow NaN values in the construction of Location -/// we can safely assume that an equivalence relation holds. -impl Eq for Location {} - -impl Ord for Location { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.partial_cmp(other) - .expect("always should return a cmp value") - } -} - -impl PartialOrd for Location { - fn partial_cmp(&self, other: &Self) -> Option { - self.0.partial_cmp(&other.0) - } -} - -impl std::hash::Hash for Location { - fn hash(&self, state: &mut H) { - let bits = self.0.to_bits(); - state.write_u64(bits); - state.finish(); - } -} - -impl TryFrom for Location { - type Error = (); - - fn try_from(value: f64) -> StdResult { - if !(0.0..=1.0).contains(&value) { - Err(()) - } else { - Ok(Location(value)) - } - } } #[derive(thiserror::Error, Debug)] @@ -529,66 +74,6 @@ pub(crate) enum RingProtoError { ConnError(#[from] conn_manager::ConnError), } -pub(crate) mod messages { - use super::*; - - #[derive(Debug, Serialize, Deserialize, Clone)] - pub(crate) enum JoinRequest { - Initial { - key: PeerKey, - hops_to_live: usize, - }, - Proxy { - joiner: PeerKeyLocation, - hops_to_live: usize, - }, - } - - #[derive(Debug, Serialize, Deserialize, Clone)] - pub(crate) enum JoinResponse { - Initial { - accepted_by: Vec, - your_location: Location, - your_peer_id: PeerKey, - }, - Proxy { - accepted_by: Vec, - }, - } - - /// A stateful connection attempt. - #[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)] - pub(crate) enum OpenConnection { - OCReceived, - Connecting, - Connected, - } - - impl OpenConnection { - pub fn is_initiated(&self) -> bool { - matches!(self, OpenConnection::Connecting) - } - - pub fn is_connected(&self) -> bool { - matches!(self, OpenConnection::Connected) - } - - pub(super) fn transition(&mut self, other_host_state: Self) { - match (*self, other_host_state) { - (Self::Connected, _) => {} - (_, Self::Connecting) => *self = Self::OCReceived, - (_, Self::OCReceived | Self::Connected) => *self = Self::Connected, - } - } - } - - impl Display for OpenConnection { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "OpenConnection::{:?}", self) - } - } -} - #[cfg(test)] mod tests { use std::collections::HashMap; @@ -604,29 +89,6 @@ mod tests { probe_proto::{self, ProbeProtocol}, }; - #[test] - fn open_connection_state_transition() { - let mut oc0 = OpenConnection::Connecting; - let oc1 = OpenConnection::Connecting; - oc0.transition(oc1); - assert_eq!(oc0, OpenConnection::OCReceived); - - let mut oc0 = OpenConnection::Connecting; - let oc1 = OpenConnection::OCReceived; - oc0.transition(oc1); - assert!(oc0.is_connected()); - - let mut oc0 = OpenConnection::Connecting; - let oc1 = OpenConnection::Connected; - oc0.transition(oc1); - assert!(oc0.is_connected()); - - let mut oc0 = OpenConnection::Connecting; - let oc1 = OpenConnection::OCReceived; - oc0.transition(oc1); - assert!(oc0.is_connected()); - } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn node0_to_gateway_conn() -> StdResult<(), Box> { //! Given a network of one node and one gateway test that both are connected. diff --git a/crates/freenet2-node/src/operations/join_ring.rs b/crates/freenet2-node/src/operations/join_ring.rs index 4444a4c5c..f8b21f4f3 100644 --- a/crates/freenet2-node/src/operations/join_ring.rs +++ b/crates/freenet2-node/src/operations/join_ring.rs @@ -1,16 +1,26 @@ use rust_fsm::*; +use super::{OpError, OperationResult}; use crate::{ - conn_manager::ConnectionBridge2, - message::{JoinRing, Message, Transaction}, + conn_manager::ConnectionBridge, + message::{Message, Transaction}, node::OpStateStorage, }; -use super::{OpError, OperationResult}; +pub(crate) use self::messages::JoinRingMsg; + +#[derive(Debug)] +pub(crate) struct JoinRingOpState(InternalJoinRingOpState); + +impl From for JoinRingOpState { + fn from(state: InternalJoinRingOpState) -> Self { + JoinRingOpState(state) + } +} state_machine! { derive(Debug) - pub(crate) JoinRingOp(Connecting) + InternalJoinRingOp(Connecting) Connecting => { Connecting => OCReceived [OCReceived], @@ -23,25 +33,26 @@ state_machine! { pub(crate) async fn join_ring( op_storage: &mut OpStateStorage, conn_manager: &mut CB, - join_op: JoinRing, + join_op: JoinRingMsg, ) -> Result<(), OpError> where - CB: ConnectionBridge2, + CB: ConnectionBridge, { let id = *join_op.id(); if let Some(state) = op_storage.pop_join_ring_op(&id) { // was an existing operation - match update_state(state, join_op) { + match update_state(state.0, join_op) { Err(tx) => { log::error!("error while processing {}", tx); - conn_manager.send(Message::Canceled(id)).await?; + conn_manager.send(Message::Canceled(tx)).await?; } Ok(OperationResult { return_msg: Some(msg), state: Some(updated_state), }) => { + // updated op conn_manager.send(msg).await?; - op_storage.push_join_ring_op(id, updated_state)?; + op_storage.push_join_ring_op(id, updated_state.into())?; } Ok(OperationResult { return_msg: Some(msg), @@ -64,37 +75,410 @@ where } fn update_state( - current_state: JoinRingOp, - other_host_msg: JoinRing, -) -> Result, Transaction> { + current_state: InternalJoinRingOpState, + other_host_msg: JoinRingMsg, +) -> Result, Transaction> { todo!() } +// fn join_ring(self: &Arc) -> Result<()> { +// if self.conn_manager.transport().is_open() && self.gateways.read().is_empty() { +// match *self.location.read() { +// Some(loc) => { +// log::info!( +// "No gateways to join through, listening for connections at loc: {}", +// loc +// ); +// return Ok(()); +// } +// None => return Err(RingProtoError::Join), +// } +// } + +// // FIXME: this iteration should be shuffled, must write an extension iterator shuffle items "in place" +// // the idea here is to limit the amount of gateways being contacted that's why shuffling is required +// for gateway in self.gateways.read().iter() { +// log::info!( +// "Joining ring via {} at {}", +// gateway.peer, +// gateway +// .location +// .ok_or(conn_manager::ConnError::LocationUnknown)? +// ); +// self.conn_manager.add_connection(*gateway, true); +// let tx = Transaction::new(::msg_type_id()); +// let join_req = messages::JoinRequest::Initial { +// key: self.peer_key, +// hops_to_live: self.max_hops_to_live, +// }; +// log::debug!("Sending {:?} to {}", join_req, gateway.peer); + +// let ring_proto = self.clone(); +// let join_response_cb = +// move |sender: PeerKeyLocation, join_res: Message| -> conn_manager::Result<()> { +// let (accepted_by, tx) = if let Message::JoinResponse( +// incoming_tx, +// messages::JoinResponse::Initial { +// accepted_by, +// your_location, +// .. +// }, +// ) = join_res +// { +// log::debug!("JoinResponse received from {}", sender.peer,); +// if incoming_tx != tx { +// return Err(conn_manager::ConnError::UnexpectedTx(tx, incoming_tx)); +// } +// let loc = &mut *ring_proto.location.write(); +// *loc = Some(your_location); +// (accepted_by, incoming_tx) +// } else { +// return Err(conn_manager::ConnError::UnexpectedResponseMessage(join_res)); +// }; + +// let self_location = &*ring_proto.location.read(); +// let self_location = +// &self_location.ok_or(conn_manager::ConnError::LocationUnknown)?; +// for new_peer_key in accepted_by { +// if ring_proto.ring.should_accept( +// self_location, +// &new_peer_key +// .location +// .ok_or(conn_manager::ConnError::LocationUnknown)?, +// ) { +// log::info!("Establishing connection to {}", new_peer_key.peer); +// ring_proto.establish_conn(new_peer_key, tx); +// } else { +// log::debug!("Not accepting connection to {}", new_peer_key.peer); +// } +// } + +// Ok(()) +// }; +// log::debug!("Initiating JoinRequest transaction: {}", tx); +// let msg: Message = (tx, join_req).into(); +// self.conn_manager +// .send_with_callback(*gateway, tx, msg, join_response_cb)?; +// } + +// Ok(()) +// } + +// fn establish_conn(self: &Arc, new_peer: PeerKeyLocation, tx: Transaction) { +// self.conn_manager.add_connection(new_peer, false); +// let self_cp = self.clone(); +// let state = Arc::new(RwLock::new(messages::OpenConnection::Connecting)); + +// let state_cp = state.clone(); +// let ack_peer = move |peer: PeerKeyLocation, msg: Message| -> conn_manager::Result<()> { +// let (tx, oc) = match msg { +// Message::OpenConnection(tx, oc) => (tx, oc), +// msg => return Err(conn_manager::ConnError::UnexpectedResponseMessage(msg)), +// }; +// let mut current_state = state_cp.write(); +// current_state.transition(oc); +// if !current_state.is_connected() { +// let open_conn: Message = (tx, *current_state).into(); +// log::debug!("Acknowledging OC"); +// self_cp +// .conn_manager +// .send(peer, *open_conn.id(), open_conn)?; +// } else { +// log::info!( +// "{} connected to {}, adding to ring", +// self_cp.peer_key, +// new_peer.peer +// ); +// self_cp.conn_manager.send( +// peer, +// tx, +// Message::from((tx, messages::OpenConnection::Connected)), +// )?; +// self_cp.ring.connections_by_location.write().insert( +// new_peer +// .location +// .ok_or(conn_manager::ConnError::LocationUnknown)?, +// new_peer, +// ); +// } +// Ok(()) +// }; +// self.conn_manager.listen_to_replies(tx, ack_peer); + +// let conn_manager = self.conn_manager.clone(); +// tokio::spawn(async move { +// let curr_time = Instant::now(); +// let mut attempts = 0; +// while !state.read().is_connected() && curr_time.elapsed() <= Duration::from_secs(30) { +// log::debug!( +// "Sending {} to {}, number of messages sent: {}", +// *state.read(), +// new_peer.peer, +// attempts +// ); +// conn_manager.send(new_peer, tx, Message::OpenConnection(tx, *state.read()))?; +// attempts += 1; +// tokio::time::sleep(Duration::from_millis(200)).await +// } +// if curr_time.elapsed() > Duration::from_secs(30) { +// log::error!("Timed out trying to connect to {}", new_peer.peer); +// Err(conn_manager::ConnError::NegotationFailed) +// } else { +// conn_manager.remove_listener(tx); +// log::info!("Success negotiating connection to {}", new_peer.peer); +// Ok(()) +// } +// }); +// } + +// fn listen_for_join_req(self: &Arc) -> ListenerHandle { +// let self_cp = self.clone(); +// let process_join_req = move |sender: PeerKeyLocation, +// msg: Message| +// -> conn_manager::Result<()> { +// let (tx, join_req) = if let Message::JoinRequest(id, join_req) = msg { +// (id, join_req) +// } else { +// return Err(conn_manager::ConnError::UnexpectedResponseMessage(msg)); +// }; + +// enum ReqType { +// Initial, +// Proxy, +// } + +// let peer_key_loc; +// let req_type; +// let jr_hpt = match join_req { +// messages::JoinRequest::Initial { key, hops_to_live } => { +// peer_key_loc = PeerKeyLocation { +// peer: key, +// location: Some(Location::random()), +// }; +// req_type = ReqType::Initial; +// hops_to_live +// } +// messages::JoinRequest::Proxy { +// joiner, +// hops_to_live, +// } => { +// peer_key_loc = joiner; +// req_type = ReqType::Proxy; +// hops_to_live +// } +// }; +// log::debug!( +// "JoinRequest received by {} with HTL {}", +// sender +// .location +// .ok_or(conn_manager::ConnError::LocationUnknown)?, +// jr_hpt +// ); + +// let your_location = self_cp +// .location +// .read() +// .ok_or(conn_manager::ConnError::LocationUnknown)?; +// let accepted_by = if self_cp.ring.should_accept( +// &your_location, +// &peer_key_loc +// .location +// .ok_or(conn_manager::ConnError::LocationUnknown)?, +// ) { +// log::debug!( +// "Accepting connections to {:?}, establising connection @ {}", +// peer_key_loc, +// self_cp.peer_key +// ); +// self_cp.establish_conn(peer_key_loc, tx); +// vec![PeerKeyLocation { +// peer: self_cp.peer_key, +// location: Some(your_location), +// }] +// } else { +// log::debug!("Not accepting new connection sender {:?}", peer_key_loc); +// Vec::new() +// }; + +// log::debug!( +// "Sending JoinResponse to {} accepting {} connections", +// sender.peer, +// accepted_by.len() +// ); +// let join_response = match req_type { +// ReqType::Initial => Message::from(( +// tx, +// JoinResponse::Initial { +// accepted_by: accepted_by.clone(), +// your_location: peer_key_loc +// .location +// .ok_or(conn_manager::ConnError::LocationUnknown)?, +// your_peer_id: peer_key_loc.peer, +// }, +// )), +// ReqType::Proxy => Message::from(( +// tx, +// JoinResponse::Proxy { +// accepted_by: accepted_by.clone(), +// }, +// )), +// }; +// self_cp.conn_manager.send(peer_key_loc, tx, join_response)?; +// // NOTE: this is in practica a jump to: join_ring.join_response_cb + +// if jr_hpt > 0 && !self_cp.ring.connections_by_location.read().is_empty() { +// let forward_to = if jr_hpt >= self_cp.rnd_if_htl_above { +// log::debug!( +// "Randomly selecting peer to forward JoinRequest sender {}", +// sender.peer +// ); +// self_cp.ring.random_peer(|p| p.peer != sender.peer) +// } else { +// log::debug!( +// "Selecting close peer to forward request sender {}", +// sender.peer +// ); +// self_cp +// .ring +// .connections_by_location +// .read() +// .get( +// &peer_key_loc +// .location +// .ok_or(conn_manager::ConnError::LocationUnknown)?, +// ) +// .filter(|it| it.peer != sender.peer) +// .copied() +// }; + +// if let Some(forward_to) = forward_to { +// let forwarded = Message::from(( +// tx, +// JoinRequest::Proxy { +// joiner: peer_key_loc, +// hops_to_live: jr_hpt.min(self_cp.max_hops_to_live) - 1, +// }, +// )); + +// let forwarded_acceptors = +// Arc::new(Mutex::new(accepted_by.into_iter().collect::>())); + +// log::debug!( +// "Forwarding JoinRequest sender {} to {}", +// sender.peer, +// forward_to.peer +// ); +// let self_cp2 = self_cp.clone(); +// let register_acceptors = +// move |jr_sender: PeerKeyLocation, join_resp| -> conn_manager::Result<()> { +// if let Message::JoinResponse(tx, resp) = join_resp { +// let new_acceptors = match resp { +// JoinResponse::Initial { accepted_by, .. } => accepted_by, +// JoinResponse::Proxy { accepted_by, .. } => accepted_by, +// }; +// let fa = &mut *forwarded_acceptors.lock(); +// new_acceptors.iter().for_each(|p| { +// if !fa.contains(p) { +// fa.insert(*p); +// } +// }); +// let msg = Message::from(( +// tx, +// JoinResponse::Proxy { +// accepted_by: new_acceptors, +// }, +// )); +// self_cp2.conn_manager.send(jr_sender, tx, msg)?; +// }; +// Ok(()) +// }; +// self_cp.conn_manager.send_with_callback( +// forward_to, +// tx, +// forwarded, +// register_acceptors, +// )?; +// } +// } +// Ok(()) +// }; +// self.conn_manager +// .listen(::msg_type_id(), process_join_req) +// } + +mod messages { + use super::*; + use crate::{conn_manager::PeerKeyLocation, ring_proto::Location, PeerKey}; + + use serde::{Deserialize, Serialize}; + + #[derive(Debug, Serialize, Deserialize, Clone)] + pub(crate) enum JoinRingMsg { + Req { id: Transaction, msg: JoinRequest }, + OC { id: Transaction, msg: JoinResponse }, + Resp { id: Transaction, msg: JoinResponse }, + } + + impl JoinRingMsg { + pub fn id(&self) -> &Transaction { + use JoinRingMsg::*; + match self { + Req { id, .. } => id, + OC { id, .. } => id, + Resp { id, .. } => id, + } + } + } + + #[derive(Debug, Serialize, Deserialize, Clone)] + pub(crate) enum JoinRequest { + Initial { + key: PeerKey, + hops_to_live: usize, + }, + Proxy { + joiner: PeerKeyLocation, + hops_to_live: usize, + }, + } + + #[derive(Debug, Serialize, Deserialize, Clone)] + pub(crate) enum JoinResponse { + Initial { + accepted_by: Vec, + your_location: Location, + your_peer_id: PeerKey, + }, + Proxy { + accepted_by: Vec, + }, + } +} + #[cfg(test)] mod tests { use super::*; - use rust_fsm::StateMachine; #[test] fn join_ring_transitions() { - let mut join_op_host_1 = StateMachine::::new(); + let mut join_op_host_1 = StateMachine::::new(); let res = join_op_host_1 - .consume(&JoinRingOpInput::Connecting) + .consume(&InternalJoinRingOpInput::Connecting) .unwrap() .unwrap(); - assert!(matches!(res, JoinRingOpOutput::OCReceived)); + assert!(matches!(res, InternalJoinRingOpOutput::OCReceived)); - let mut join_op_host_2 = StateMachine::::new(); + let mut join_op_host_2 = StateMachine::::new(); let res = join_op_host_2 - .consume(&JoinRingOpInput::OCReceived) + .consume(&InternalJoinRingOpInput::OCReceived) .unwrap() .unwrap(); - assert!(matches!(res, JoinRingOpOutput::Connected)); + assert!(matches!(res, InternalJoinRingOpOutput::Connected)); let res = join_op_host_1 - .consume(&JoinRingOpInput::Connected) + .consume(&InternalJoinRingOpInput::Connected) .unwrap() .unwrap(); - assert!(matches!(res, JoinRingOpOutput::Connected)); + assert!(matches!(res, InternalJoinRingOpOutput::Connected)); } } diff --git a/crates/freenet2-node/src/ring_proto.rs b/crates/freenet2-node/src/ring_proto.rs index 672d78923..db911ef92 100644 --- a/crates/freenet2-node/src/ring_proto.rs +++ b/crates/freenet2-node/src/ring_proto.rs @@ -1,15 +1,66 @@ -// #![allow(unused)] // FIXME: remove this attr +//! Ring protocol logic and supporting types. -use std::{convert::TryFrom, fmt::Display, hash::Hasher}; +use std::{collections::BTreeMap, convert::TryFrom, fmt::Display, hash::Hasher}; -use serde::{Deserialize, Serialize}; +use parking_lot::RwLock; -use crate::{ - conn_manager::{self, PeerKey, PeerKeyLocation}, - StdResult, -}; +use crate::conn_manager::{self, PeerKeyLocation}; -type Result = StdResult; +#[derive(Debug)] +pub(crate) struct Ring { + pub connections_by_location: RwLock>, +} + +impl Ring { + const MIN_CONNECTIONS: usize = 10; + const MAX_CONNECTIONS: usize = 20; + + fn new() -> Self { + Ring { + connections_by_location: RwLock::new(BTreeMap::new()), + } + } + + fn should_accept(&self, my_location: &Location, location: &Location) -> bool { + let cbl = &*self.connections_by_location.read(); + if location == my_location || cbl.contains_key(location) { + false + } else if cbl.len() < Self::MIN_CONNECTIONS { + true + } else if cbl.len() >= Self::MAX_CONNECTIONS { + false + } else { + my_location.distance(location) < self.median_distance_to(my_location) + } + } + + fn median_distance_to(&self, location: &Location) -> Distance { + let mut conn_by_dist = self.connections_by_distance(location); + conn_by_dist.sort_by_key(|(k, _)| *k); + let idx = self.connections_by_location.read().len() / 2; + conn_by_dist[idx].0 + } + + pub fn connections_by_distance(&self, to: &Location) -> Vec<(Distance, PeerKeyLocation)> { + self.connections_by_location + .read() + .iter() + .map(|(key, peer)| (key.distance(to), *peer)) + .collect() + } + + fn random_peer(&self, filter_fn: F) -> Option + where + F: FnMut(&&PeerKeyLocation) -> bool, + { + // FIXME: should be optimized and avoid copying + self.connections_by_location + .read() + .values() + .find(filter_fn) + .copied() + } +} /// An abstract location on the 1D ring, represented by a real number on the interal [0, 1] #[derive(Debug, serde::Serialize, serde::Deserialize, Clone, Copy)] @@ -77,7 +128,7 @@ impl std::hash::Hash for Location { impl TryFrom for Location { type Error = (); - fn try_from(value: f64) -> StdResult { + fn try_from(value: f64) -> Result { if !(0.0..=1.0).contains(&value) { Err(()) } else { @@ -93,63 +144,3 @@ pub(crate) enum RingProtoError { #[error(transparent)] ConnError(#[from] conn_manager::ConnError), } - -pub(crate) mod messages { - use super::*; - - #[derive(Debug, Serialize, Deserialize, Clone)] - pub(crate) enum JoinRequest { - Initial { - key: PeerKey, - hops_to_live: usize, - }, - Proxy { - joiner: PeerKeyLocation, - hops_to_live: usize, - }, - } - - #[derive(Debug, Serialize, Deserialize, Clone)] - pub(crate) enum JoinResponse { - Initial { - accepted_by: Vec, - your_location: Location, - your_peer_id: PeerKey, - }, - Proxy { - accepted_by: Vec, - }, - } - - /// A stateful connection attempt. - #[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)] - pub(crate) enum OpenConnection { - OCReceived, - Connecting, - Connected, - } - - impl OpenConnection { - pub fn is_initiated(&self) -> bool { - matches!(self, OpenConnection::Connecting) - } - - pub fn is_connected(&self) -> bool { - matches!(self, OpenConnection::Connected) - } - - pub(super) fn transition(&mut self, other_host_state: Self) { - match (*self, other_host_state) { - (Self::Connected, _) => {} - (_, Self::Connecting) => *self = Self::OCReceived, - (_, Self::OCReceived | Self::Connected) => *self = Self::Connected, - } - } - } - - impl Display for OpenConnection { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "OpenConnection::{:?}", self) - } - } -} From 3e810e2e2e4cfdb986d56cc050699292ed3f9625 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Sat, 18 Sep 2021 14:07:43 +0200 Subject: [PATCH 03/14] Initialize join request through op --- crates/freenet2-node/src/conn_manager.rs | 2 + .../src/conn_manager/in_memory.rs | 4 + crates/freenet2-node/src/node.rs | 2 +- crates/freenet2-node/src/node/in_memory.rs | 36 +- crates/freenet2-node/src/node/op_state.rs | 12 +- crates/freenet2-node/src/operations.rs | 12 +- .../freenet2-node/src/operations/join_ring.rs | 399 +++++++++++------- 7 files changed, 305 insertions(+), 162 deletions(-) diff --git a/crates/freenet2-node/src/conn_manager.rs b/crates/freenet2-node/src/conn_manager.rs index ced683907..e3cc5486c 100644 --- a/crates/freenet2-node/src/conn_manager.rs +++ b/crates/freenet2-node/src/conn_manager.rs @@ -38,6 +38,8 @@ impl Default for ListenerHandle { #[async_trait::async_trait] pub(crate) trait ConnectionBridge { + fn add_connection(&mut self, peer: PeerKeyLocation, unsolicited: bool); + async fn recv(&self) -> Result; async fn send(&self, msg: Message) -> Result<()>; diff --git a/crates/freenet2-node/src/conn_manager/in_memory.rs b/crates/freenet2-node/src/conn_manager/in_memory.rs index 430a375b5..17f057107 100644 --- a/crates/freenet2-node/src/conn_manager/in_memory.rs +++ b/crates/freenet2-node/src/conn_manager/in_memory.rs @@ -114,6 +114,10 @@ impl ConnectionBridge for MemoryConnManager { async fn send(&self, msg: Message) -> Result<(), ConnError> { todo!() } + + fn add_connection(&mut self, peer: PeerKeyLocation, unsolicited: bool) { + todo!() + } } // impl ConnectionBridge for MemoryConnManager { diff --git a/crates/freenet2-node/src/node.rs b/crates/freenet2-node/src/node.rs index e5978a272..75f116f99 100644 --- a/crates/freenet2-node/src/node.rs +++ b/crates/freenet2-node/src/node.rs @@ -5,7 +5,7 @@ use libp2p::{identity, multiaddr::Protocol, Multiaddr, PeerId}; use crate::config::CONF; use self::{in_memory::InMemory, libp2p_impl::NodeLibP2P}; -pub(crate) use op_state::{OpStateError, OpStateStorage}; +pub(crate) use op_state::{OpExecutionError, OpStateStorage}; mod in_memory; mod libp2p_impl; diff --git a/crates/freenet2-node/src/node/in_memory.rs b/crates/freenet2-node/src/node/in_memory.rs index 07cd3f750..5f5239bfb 100644 --- a/crates/freenet2-node/src/node/in_memory.rs +++ b/crates/freenet2-node/src/node/in_memory.rs @@ -1,7 +1,7 @@ use crate::{ - conn_manager::{in_memory::MemoryConnManager, ConnectionBridge}, + conn_manager::{in_memory::MemoryConnManager, ConnectionBridge, PeerKeyLocation}, message::Message, - operations::join_ring, + operations::join_ring::{self, JoinRingOp}, NodeConfig, PeerKey, }; @@ -12,6 +12,7 @@ pub(super) struct InMemory { listening: bool, conn_manager: MemoryConnManager, op_storage: OpStateStorage, + gateways: Vec, } impl InMemory { @@ -28,9 +29,38 @@ impl InMemory { listening: true, conn_manager, op_storage: OpStateStorage::new(), + gateways: vec![], }) } + pub async fn start(&mut self) -> Result<(), ()> { + if self.listening { + // cannot start if already listening; meaning that this node already joined + // the ring or is a gateway + return Err(()); + } + + // FIXME: this iteration should be shuffled, must write an extension iterator shuffle items "in place" + // the idea here is to limit the amount of gateways being contacted that's why shuffling is required + for gateway in &self.gateways { + // initiate join action action per each gateway + let op = JoinRingOp::new( + PeerKeyLocation { + peer: self.peer, + location: None, + }, + *gateway, + ); + join_ring::initial_join_request(&mut self.op_storage, &mut self.conn_manager, op) + .await + .unwrap(); + } + + Err(()) + } + + /// Starts listening to incoming messages, only allowed to be called directly when this node + /// already joined the network. pub async fn listen_on(&mut self) -> Result<(), ()> { if !self.listening { return Err(()); @@ -44,10 +74,10 @@ impl InMemory { .await .unwrap(); } + Message::Canceled(_) => todo!(), // old: Message::ProbeRequest(_, _) => todo!(), Message::ProbeResponse(_, _) => todo!(), - Message::Canceled(_) => todo!(), }, Err(_) => break Err(()), } diff --git a/crates/freenet2-node/src/node/op_state.rs b/crates/freenet2-node/src/node/op_state.rs index 088d3cc7d..46adb6ae4 100644 --- a/crates/freenet2-node/src/node/op_state.rs +++ b/crates/freenet2-node/src/node/op_state.rs @@ -15,10 +15,10 @@ impl OpStateStorage { pub fn push_join_ring_op( &mut self, id: Transaction, - tx: join_ring::JoinRingOpState, - ) -> Result<(), OpStateError> { + tx: join_ring::JoinRingOp, + ) -> Result<(), OpExecutionError> { if !matches!(id.tx_type(), TransactionTypeId::JoinRing) { - return Err(OpStateError::IncorrectTxType( + return Err(OpExecutionError::IncorrectTxType( TransactionTypeId::JoinRing, id.tx_type(), )); @@ -27,15 +27,17 @@ impl OpStateStorage { Ok(()) } - pub fn pop_join_ring_op(&mut self, id: &Transaction) -> Option { + pub fn pop_join_ring_op(&mut self, id: &Transaction) -> Option { self.ops.join_ring.remove(id) } } #[derive(Debug, thiserror::Error)] -pub(crate) enum OpStateError { +pub(crate) enum OpExecutionError { #[error("unspected transaction type, trying to get a {0:?} from a {1:?}")] IncorrectTxType(TransactionTypeId, TransactionTypeId), + #[error("failed while processing transaction")] + UpdateFailure, } #[cfg(test)] diff --git a/crates/freenet2-node/src/operations.rs b/crates/freenet2-node/src/operations.rs index 29b89bce1..30f1dbc24 100644 --- a/crates/freenet2-node/src/operations.rs +++ b/crates/freenet2-node/src/operations.rs @@ -3,10 +3,10 @@ use serde::{Deserialize, Serialize}; use crate::{ conn_manager, message::{Message, Transaction}, - node::OpStateError, + node::OpExecutionError, }; -use self::join_ring::JoinRingOpState; +use self::join_ring::JoinRingOp; pub(crate) mod join_ring; @@ -25,7 +25,9 @@ pub(crate) enum OpError { #[error(transparent)] ConnError(#[from] conn_manager::ConnError), #[error(transparent)] - OpStateManagerError(#[from] OpStateError), + OpStateManagerError(#[from] OpExecutionError), + #[error("illegal awaiting state")] + IllegalStateTransition, } macro_rules! op_type_enumeration { @@ -36,8 +38,6 @@ macro_rules! op_type_enumeration { $($var,)+ } - - #[derive(Debug)] pub(crate) struct OpsMap { $( pub $field: std::collections::HashMap),+, } @@ -53,6 +53,6 @@ macro_rules! op_type_enumeration { } op_type_enumeration!(decl struct { - join_ring: JoinRingOpState, + join_ring: JoinRingOp, probe_peers: ProbeOp }); diff --git a/crates/freenet2-node/src/operations/join_ring.rs b/crates/freenet2-node/src/operations/join_ring.rs index f8b21f4f3..2301e083d 100644 --- a/crates/freenet2-node/src/operations/join_ring.rs +++ b/crates/freenet2-node/src/operations/join_ring.rs @@ -2,34 +2,124 @@ use rust_fsm::*; use super::{OpError, OperationResult}; use crate::{ - conn_manager::ConnectionBridge, - message::{Message, Transaction}, - node::OpStateStorage, + conn_manager::{self, ConnectionBridge, PeerKeyLocation}, + message::{Message, Transaction, TransactionType}, + node::{OpExecutionError, OpStateStorage}, }; pub(crate) use self::messages::JoinRingMsg; +pub(crate) struct JoinRingOp(StateMachine); + +impl JoinRingOp { + pub fn new(this_peer: PeerKeyLocation, gateway: PeerKeyLocation) -> Self { + let mut machine = StateMachine::new(); + machine + .consume(&JRInput::Connecting { gateway, this_peer }) + .expect(""); + JoinRingOp(machine) + } +} + #[derive(Debug)] -pub(crate) struct JoinRingOpState(InternalJoinRingOpState); +struct InternalJROp; + +impl StateMachineImpl for InternalJROp { + type Input = JRInput; + + type State = JRState; + + type Output = JROutput; + + const INITIAL_STATE: Self::State = JRState::Initializing; + + fn transition(state: &Self::State, input: &Self::Input) -> Option { + match (state, input) { + ( + JRState::Initializing, + JRInput::Connecting { + gateway, this_peer, .. + }, + ) => Some(JRState::Connecting(ConnectionInfo { + gateway: *gateway, + this_peer: *this_peer, + max_hops_to_live: todo!(), + })), + (JRState::Connecting { .. }, JRInput::Connecting { .. }) => Some(JRState::OCReceived), + (JRState::Connecting { .. }, JRInput::OCReceived | JRInput::Connected) => { + Some(JRState::Connected) + } + (JRState::OCReceived { .. }, JRInput::Connected) => Some(JRState::Connected), + (JRState::Connected, _) => None, + _ => None, + } + } -impl From for JoinRingOpState { - fn from(state: InternalJoinRingOpState) -> Self { - JoinRingOpState(state) + fn output(state: &Self::State, input: &Self::Input) -> Option { + match (state, input) { + (JRState::Initializing, JRInput::Connecting { this_peer, .. }) => { + Some(JROutput::OCReceived { + by_peer: *this_peer, + }) + } + (JRState::Connecting { .. }, JRInput::Connecting { this_peer, .. }) => { + Some(JROutput::OCReceived { + by_peer: *this_peer, + }) + } + (JRState::Connecting { .. }, JRInput::OCReceived | JRInput::Connected) => { + Some(JROutput::Connected) + } + (JRState::OCReceived, JRInput::Connected) => Some(JROutput::Connected), + _ => None, + } } } -state_machine! { - derive(Debug) - InternalJoinRingOp(Connecting) +#[derive(Debug, Clone)] +enum JRState { + Initializing, + Connecting(ConnectionInfo), + OCReceived, + Connected, +} + +#[derive(Debug, Clone)] +struct ConnectionInfo { + gateway: PeerKeyLocation, + this_peer: PeerKeyLocation, + max_hops_to_live: usize, +} - Connecting => { - Connecting => OCReceived [OCReceived], - OCReceived => Connected [Connected], - Connected => Connected [Connected], +impl JRState { + fn try_unwrap_connecting(self) -> Result { + if let Self::Connecting(conn_info) = self { + Ok(conn_info) + } else { + Err(OpExecutionError::UpdateFailure.into()) + } + } +} + +enum JRInput { + Connecting { + gateway: PeerKeyLocation, + this_peer: PeerKeyLocation, }, - OCReceived(Connected) => Connected [Connected], + OCReceived, + Connected, +} + +enum JROutput { + OCReceived { by_peer: PeerKeyLocation }, + Connected, } +/// Join ring routine, called upon processing a request to join or performing +/// a join operation for this node. +/// +/// # Arguments +/// - join_op: no nodes pub(crate) async fn join_ring( op_storage: &mut OpStateStorage, conn_manager: &mut CB, @@ -38,131 +128,140 @@ pub(crate) async fn join_ring( where CB: ConnectionBridge, { - let id = *join_op.id(); - if let Some(state) = op_storage.pop_join_ring_op(&id) { - // was an existing operation - match update_state(state.0, join_op) { - Err(tx) => { - log::error!("error while processing {}", tx); - conn_manager.send(Message::Canceled(tx)).await?; - } - Ok(OperationResult { - return_msg: Some(msg), - state: Some(updated_state), - }) => { - // updated op - conn_manager.send(msg).await?; - op_storage.push_join_ring_op(id, updated_state.into())?; - } - Ok(OperationResult { - return_msg: Some(msg), - state: None, - }) => { - // finished the operation at this node, informing back - conn_manager.send(msg).await?; - } - Ok(OperationResult { - return_msg: None, - state: None, - }) => { - // operation finished_completely - } - _ => unreachable!(), - } + let result = if let Some(state) = op_storage.pop_join_ring_op(join_op.id()) { + // was an existing operation, the other peer messaged back + update_state(state, join_op) } else { + // new request to join from this node, initialize the machine + let machine = JoinRingOp(StateMachine::new()); + update_state(machine, join_op) + }; + + match result.map_err(|_| OpExecutionError::UpdateFailure) { + Err(tx) => { + log::error!("error while processing join request: {}", tx); + let tx = todo!(); + conn_manager.send(Message::Canceled(tx)).await?; + return Err(OpExecutionError::UpdateFailure.into()); + } + Ok(OperationResult { + return_msg: Some(msg), + state: Some(updated_state), + }) => { + // updated op + let id = *msg.id(); + conn_manager.send(msg).await?; + op_storage.push_join_ring_op(id, updated_state)?; + } + Ok(OperationResult { + return_msg: Some(msg), + state: None, + }) => { + // finished the operation at this node, informing back + conn_manager.send(msg).await?; + } + Ok(OperationResult { + return_msg: None, + state: None, + }) => { + // operation finished_completely + } + _ => unreachable!(), } Ok(()) } +#[inline(always)] fn update_state( - current_state: InternalJoinRingOpState, + current_state: JoinRingOp, other_host_msg: JoinRingMsg, -) -> Result, Transaction> { +) -> Result, ()> { + // let join_response_cb = + // move |sender: PeerKeyLocation, join_res: Message| -> conn_manager::Result<()> { + // let (accepted_by, tx) = if let Message::JoinResponse( + // incoming_tx, + // messages::JoinResponse::Initial { + // accepted_by, + // your_location, + // .. + // }, + // ) = join_res + // { + // log::debug!("JoinResponse received from {}", sender.peer,); + // if incoming_tx != tx { + // return Err(conn_manager::ConnError::UnexpectedTx(tx, incoming_tx)); + // } + // let loc = &mut *ring_proto.location.write(); + // *loc = Some(your_location); + // (accepted_by, incoming_tx) + // } else { + // return Err(conn_manager::ConnError::UnexpectedResponseMessage(join_res)); + // }; + // let self_location = &*ring_proto.location.read(); + // let self_location = + // &self_location.ok_or(conn_manager::ConnError::LocationUnknown)?; + // for new_peer_key in accepted_by { + // if ring_proto.ring.should_accept( + // self_location, + // &new_peer_key + // .location + // .ok_or(conn_manager::ConnError::LocationUnknown)?, + // ) { + // log::info!("Establishing connection to {}", new_peer_key.peer); + // ring_proto.establish_conn(new_peer_key, tx); + // } else { + // log::debug!("Not accepting connection to {}", new_peer_key.peer); + // } + // } + // Ok(()) + // }; todo!() } -// fn join_ring(self: &Arc) -> Result<()> { -// if self.conn_manager.transport().is_open() && self.gateways.read().is_empty() { -// match *self.location.read() { -// Some(loc) => { -// log::info!( -// "No gateways to join through, listening for connections at loc: {}", -// loc -// ); -// return Ok(()); -// } -// None => return Err(RingProtoError::Join), -// } -// } - -// // FIXME: this iteration should be shuffled, must write an extension iterator shuffle items "in place" -// // the idea here is to limit the amount of gateways being contacted that's why shuffling is required -// for gateway in self.gateways.read().iter() { -// log::info!( -// "Joining ring via {} at {}", -// gateway.peer, -// gateway -// .location -// .ok_or(conn_manager::ConnError::LocationUnknown)? -// ); -// self.conn_manager.add_connection(*gateway, true); -// let tx = Transaction::new(::msg_type_id()); -// let join_req = messages::JoinRequest::Initial { -// key: self.peer_key, -// hops_to_live: self.max_hops_to_live, -// }; -// log::debug!("Sending {:?} to {}", join_req, gateway.peer); - -// let ring_proto = self.clone(); -// let join_response_cb = -// move |sender: PeerKeyLocation, join_res: Message| -> conn_manager::Result<()> { -// let (accepted_by, tx) = if let Message::JoinResponse( -// incoming_tx, -// messages::JoinResponse::Initial { -// accepted_by, -// your_location, -// .. -// }, -// ) = join_res -// { -// log::debug!("JoinResponse received from {}", sender.peer,); -// if incoming_tx != tx { -// return Err(conn_manager::ConnError::UnexpectedTx(tx, incoming_tx)); -// } -// let loc = &mut *ring_proto.location.write(); -// *loc = Some(your_location); -// (accepted_by, incoming_tx) -// } else { -// return Err(conn_manager::ConnError::UnexpectedResponseMessage(join_res)); -// }; - -// let self_location = &*ring_proto.location.read(); -// let self_location = -// &self_location.ok_or(conn_manager::ConnError::LocationUnknown)?; -// for new_peer_key in accepted_by { -// if ring_proto.ring.should_accept( -// self_location, -// &new_peer_key -// .location -// .ok_or(conn_manager::ConnError::LocationUnknown)?, -// ) { -// log::info!("Establishing connection to {}", new_peer_key.peer); -// ring_proto.establish_conn(new_peer_key, tx); -// } else { -// log::debug!("Not accepting connection to {}", new_peer_key.peer); -// } -// } - -// Ok(()) -// }; -// log::debug!("Initiating JoinRequest transaction: {}", tx); -// let msg: Message = (tx, join_req).into(); -// self.conn_manager -// .send_with_callback(*gateway, tx, msg, join_response_cb)?; -// } - -// Ok(()) -// } +pub(crate) async fn initial_join_request( + op_storage: &mut OpStateStorage, + conn_manager: &mut CB, + mut join_op: JoinRingOp, +) -> Result<(), OpError> +where + CB: ConnectionBridge, +{ + let ConnectionInfo { + gateway, + this_peer, + max_hops_to_live, + } = (&join_op.0).state().clone().try_unwrap_connecting()?; + + log::info!( + "Joining ring via {} at {}", + gateway.peer, + gateway + .location + .ok_or(conn_manager::ConnError::LocationUnknown)? + ); + + conn_manager.add_connection(gateway, true); + let tx = Transaction::new(::msg_type_id()); + let join_req = Message::from(messages::JoinRingMsg::Req { + id: tx, + msg: messages::JoinRequest::Initial { + key: this_peer.peer, + hops_to_live: max_hops_to_live, + }, + }); + log::debug!( + "Sending initial join tx: {:?} to {}", + join_req, + gateway.peer + ); + conn_manager.send(join_req).await?; + join_op + .0 + .consume(&JRInput::Connecting { gateway, this_peer }) + .map_err(|_| OpError::IllegalStateTransition)?; + op_storage.push_join_ring_op(tx, join_op)?; + Ok(()) +} // fn establish_conn(self: &Arc, new_peer: PeerKeyLocation, tx: Transaction) { // self.conn_manager.add_connection(new_peer, false); @@ -204,7 +303,6 @@ fn update_state( // Ok(()) // }; // self.conn_manager.listen_to_replies(tx, ack_peer); - // let conn_manager = self.conn_manager.clone(); // tokio::spawn(async move { // let curr_time = Instant::now(); @@ -241,12 +339,10 @@ fn update_state( // } else { // return Err(conn_manager::ConnError::UnexpectedResponseMessage(msg)); // }; - // enum ReqType { // Initial, // Proxy, // } - // let peer_key_loc; // let req_type; // let jr_hpt = match join_req { @@ -274,7 +370,6 @@ fn update_state( // .ok_or(conn_manager::ConnError::LocationUnknown)?, // jr_hpt // ); - // let your_location = self_cp // .location // .read() @@ -299,7 +394,6 @@ fn update_state( // log::debug!("Not accepting new connection sender {:?}", peer_key_loc); // Vec::new() // }; - // log::debug!( // "Sending JoinResponse to {} accepting {} connections", // sender.peer, @@ -325,7 +419,6 @@ fn update_state( // }; // self_cp.conn_manager.send(peer_key_loc, tx, join_response)?; // // NOTE: this is in practica a jump to: join_ring.join_response_cb - // if jr_hpt > 0 && !self_cp.ring.connections_by_location.read().is_empty() { // let forward_to = if jr_hpt >= self_cp.rnd_if_htl_above { // log::debug!( @@ -350,7 +443,6 @@ fn update_state( // .filter(|it| it.peer != sender.peer) // .copied() // }; - // if let Some(forward_to) = forward_to { // let forwarded = Message::from(( // tx, @@ -359,10 +451,8 @@ fn update_state( // hops_to_live: jr_hpt.min(self_cp.max_hops_to_live) - 1, // }, // )); - // let forwarded_acceptors = // Arc::new(Mutex::new(accepted_by.into_iter().collect::>())); - // log::debug!( // "Forwarding JoinRequest sender {} to {}", // sender.peer, @@ -457,28 +547,43 @@ mod messages { #[cfg(test)] mod tests { + use libp2p::identity::Keypair; + use super::*; + use crate::PeerKey; #[test] fn join_ring_transitions() { - let mut join_op_host_1 = StateMachine::::new(); + let h1 = PeerKeyLocation { + peer: PeerKey::from(Keypair::generate_ed25519().public()), + location: None, + }; + let h2 = PeerKeyLocation { + peer: PeerKey::from(Keypair::generate_ed25519().public()), + location: None, + }; + + let mut join_op_host_1 = StateMachine::::new(); let res = join_op_host_1 - .consume(&InternalJoinRingOpInput::Connecting) + .consume(&JRInput::Connecting { + gateway: h2, + this_peer: h1, + }) .unwrap() .unwrap(); - assert!(matches!(res, InternalJoinRingOpOutput::OCReceived)); + assert!(matches!(res, JROutput::OCReceived { by_peer: h1 })); - let mut join_op_host_2 = StateMachine::::new(); + let mut join_op_host_2 = StateMachine::::new(); let res = join_op_host_2 - .consume(&InternalJoinRingOpInput::OCReceived) + .consume(&JRInput::OCReceived) .unwrap() .unwrap(); - assert!(matches!(res, InternalJoinRingOpOutput::Connected)); + assert!(matches!(res, JROutput::Connected)); let res = join_op_host_1 - .consume(&InternalJoinRingOpInput::Connected) + .consume(&JRInput::Connected) .unwrap() .unwrap(); - assert!(matches!(res, InternalJoinRingOpOutput::Connected)); + assert!(matches!(res, JROutput::Connected)); } } From 5be0f9da6570d75b9bafcd77e02b9e87b87eab1a Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Wed, 22 Sep 2021 22:16:39 +0200 Subject: [PATCH 04/14] Add update_state func --- crates/freenet2-node/src/conn_manager.rs | 4 +- .../src/conn_manager/in_memory.rs | 4 +- crates/freenet2-node/src/lib.rs | 2 +- crates/freenet2-node/src/message.rs | 12 +- crates/freenet2-node/src/node/op_state.rs | 13 +- .../freenet2-node/src/operations/join_ring.rs | 643 ++++++++++-------- .../src/{ring_proto.rs => ring.rs} | 31 +- 7 files changed, 402 insertions(+), 307 deletions(-) rename crates/freenet2-node/src/{ring_proto.rs => ring.rs} (78%) diff --git a/crates/freenet2-node/src/conn_manager.rs b/crates/freenet2-node/src/conn_manager.rs index e3cc5486c..47befa80e 100644 --- a/crates/freenet2-node/src/conn_manager.rs +++ b/crates/freenet2-node/src/conn_manager.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Deserializer, Serialize, Serializer}; use crate::{ message::{Message, Transaction}, - ring_proto::Location, + ring::Location, StdResult, }; @@ -42,7 +42,7 @@ pub(crate) trait ConnectionBridge { async fn recv(&self) -> Result; - async fn send(&self, msg: Message) -> Result<()>; + async fn send(&self, target: &PeerKeyLocation, msg: Message) -> Result<()>; } /// A protocol used to send and receive data over the network. diff --git a/crates/freenet2-node/src/conn_manager/in_memory.rs b/crates/freenet2-node/src/conn_manager/in_memory.rs index 17f057107..c77880535 100644 --- a/crates/freenet2-node/src/conn_manager/in_memory.rs +++ b/crates/freenet2-node/src/conn_manager/in_memory.rs @@ -10,7 +10,7 @@ use crate::{ config::tracing::Logger, conn_manager::{self, ConnectionBridge, ListenerHandle, PeerKey, PeerKeyLocation}, message::{Message, Transaction, TransactionTypeId}, - ring_proto::Location, + ring::Location, }; type InboundListenerFn = @@ -111,7 +111,7 @@ impl ConnectionBridge for MemoryConnManager { todo!() } - async fn send(&self, msg: Message) -> Result<(), ConnError> { + async fn send(&self, target: &PeerKeyLocation, msg: Message) -> Result<(), ConnError> { todo!() } diff --git a/crates/freenet2-node/src/lib.rs b/crates/freenet2-node/src/lib.rs index 165206917..58c040865 100644 --- a/crates/freenet2-node/src/lib.rs +++ b/crates/freenet2-node/src/lib.rs @@ -4,7 +4,7 @@ mod message; mod node; mod operations; // mod probe_proto; -mod ring_proto; +mod ring; pub use conn_manager::PeerKey; pub use node::NodeConfig; diff --git a/crates/freenet2-node/src/message.rs b/crates/freenet2-node/src/message.rs index 57e195093..562d5f162 100644 --- a/crates/freenet2-node/src/message.rs +++ b/crates/freenet2-node/src/message.rs @@ -3,7 +3,7 @@ use std::{fmt::Display, time::Duration}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::{operations::join_ring::JoinRingMsg, ring_proto::Location}; +use crate::{conn_manager::PeerKeyLocation, operations::join_ring::JoinRingMsg, ring::Location}; pub(crate) use sealed_msg_type::TransactionTypeId; /// An transaction is a unique, universal and efficient identifier for any @@ -128,6 +128,16 @@ impl Message { } } + pub fn sender(&self) -> Option<&PeerKeyLocation> { + use Message::*; + match self { + JoinRing(op) => op.sender(), + ProbeRequest(id, _) => None, + ProbeResponse(id, _) => None, + Canceled(_) => None, + } + } + pub fn msg_type(&self) -> TransactionTypeId { todo!() } diff --git a/crates/freenet2-node/src/node/op_state.rs b/crates/freenet2-node/src/node/op_state.rs index 46adb6ae4..5c33c1044 100644 --- a/crates/freenet2-node/src/node/op_state.rs +++ b/crates/freenet2-node/src/node/op_state.rs @@ -1,15 +1,22 @@ +use std::sync::Arc; + use crate::{ message::{Transaction, TransactionTypeId}, operations::{join_ring, OpsMap}, + ring::Ring, }; pub(crate) struct OpStateStorage { ops: OpsMap, + pub ring: Arc, } impl OpStateStorage { pub fn new() -> Self { - Self { ops: OpsMap::new() } + Self { + ops: OpsMap::new(), + ring: Arc::new(Ring::new()), + } } pub fn push_join_ring_op( @@ -36,8 +43,8 @@ impl OpStateStorage { pub(crate) enum OpExecutionError { #[error("unspected transaction type, trying to get a {0:?} from a {1:?}")] IncorrectTxType(TransactionTypeId, TransactionTypeId), - #[error("failed while processing transaction")] - UpdateFailure, + #[error("failed while processing transaction {0}")] + TxUpdateFailure(Transaction), } #[cfg(test)] diff --git a/crates/freenet2-node/src/operations/join_ring.rs b/crates/freenet2-node/src/operations/join_ring.rs index 2301e083d..750a5f578 100644 --- a/crates/freenet2-node/src/operations/join_ring.rs +++ b/crates/freenet2-node/src/operations/join_ring.rs @@ -1,3 +1,5 @@ +use std::collections::HashSet; + use rust_fsm::*; use super::{OpError, OperationResult}; @@ -5,18 +7,17 @@ use crate::{ conn_manager::{self, ConnectionBridge, PeerKeyLocation}, message::{Message, Transaction, TransactionType}, node::{OpExecutionError, OpStateStorage}, + ring::{Location, Ring}, + PeerKey, }; -pub(crate) use self::messages::JoinRingMsg; +pub(crate) use self::messages::{JoinRequest, JoinResponse, JoinRingMsg}; pub(crate) struct JoinRingOp(StateMachine); impl JoinRingOp { pub fn new(this_peer: PeerKeyLocation, gateway: PeerKeyLocation) -> Self { let mut machine = StateMachine::new(); - machine - .consume(&JRInput::Connecting { gateway, this_peer }) - .expect(""); JoinRingOp(machine) } } @@ -25,11 +26,11 @@ impl JoinRingOp { struct InternalJROp; impl StateMachineImpl for InternalJROp { - type Input = JRInput; + type Input = JoinRingMsg; type State = JRState; - type Output = JROutput; + type Output = JoinRingMsg; const INITIAL_STATE: Self::State = JRState::Initializing; @@ -37,19 +38,27 @@ impl StateMachineImpl for InternalJROp { match (state, input) { ( JRState::Initializing, - JRInput::Connecting { - gateway, this_peer, .. + JoinRingMsg::Req { + id, + msg: + JoinRequest::Initial { + req_peer, + target_loc, + max_hops_to_live, + .. + }, }, ) => Some(JRState::Connecting(ConnectionInfo { - gateway: *gateway, - this_peer: *this_peer, - max_hops_to_live: todo!(), + gateway: *target_loc, + this_peer: *req_peer, + max_hops_to_live: *max_hops_to_live, })), - (JRState::Connecting { .. }, JRInput::Connecting { .. }) => Some(JRState::OCReceived), - (JRState::Connecting { .. }, JRInput::OCReceived | JRInput::Connected) => { - Some(JRState::Connected) - } - (JRState::OCReceived { .. }, JRInput::Connected) => Some(JRState::Connected), + // (JRState::Connecting { .. }, JRInput::Connecting { .. }) => Some(JRState::OCReceived), + ( + JRState::Connecting { .. }, + JoinRingMsg::Req { .. } | JoinRingMsg::Connected { .. }, + ) => Some(JRState::Connected), + (JRState::OCReceived { .. }, JoinRingMsg::Connected) => Some(JRState::Connected), (JRState::Connected, _) => None, _ => None, } @@ -57,20 +66,43 @@ impl StateMachineImpl for InternalJROp { fn output(state: &Self::State, input: &Self::Input) -> Option { match (state, input) { - (JRState::Initializing, JRInput::Connecting { this_peer, .. }) => { - Some(JROutput::OCReceived { - by_peer: *this_peer, - }) - } - (JRState::Connecting { .. }, JRInput::Connecting { this_peer, .. }) => { - Some(JROutput::OCReceived { - by_peer: *this_peer, - }) - } - (JRState::Connecting { .. }, JRInput::OCReceived | JRInput::Connected) => { - Some(JROutput::Connected) - } - (JRState::OCReceived, JRInput::Connected) => Some(JROutput::Connected), + ( + JRState::Initializing, + JoinRingMsg::Req { + id, + msg: + JoinRequest::Initial { + target_loc, + req_peer, + .. + }, + }, + ) => Some(JoinRingMsg::Resp { + id: *id, + msg: JoinResponse::ReceivedOC { + by_peer: *target_loc, + }, + sender: PeerKeyLocation { + peer: *req_peer, + location: None, + }, + }), + ( + JRState::Connecting { .. }, + JoinRingMsg::Resp { + id, + sender, + msg: JoinResponse::ReceivedOC { by_peer }, + }, + ) => Some(JoinRingMsg::Resp { + id: *id, + msg: JoinResponse::ReceivedOC { by_peer: *by_peer }, + sender: *sender, + }), + // (JRState::Connecting { .. }, JRInput::OCReceived | JRInput::Connected) => { + // Some(JROutput::Connected) + // } + // (JRState::OCReceived, &JoinRingMsg::OC) => Some(JROutput::Connected), _ => None, } } @@ -87,7 +119,7 @@ enum JRState { #[derive(Debug, Clone)] struct ConnectionInfo { gateway: PeerKeyLocation, - this_peer: PeerKeyLocation, + this_peer: PeerKey, max_hops_to_live: usize, } @@ -96,7 +128,7 @@ impl JRState { if let Self::Connecting(conn_info) = self { Ok(conn_info) } else { - Err(OpExecutionError::UpdateFailure.into()) + Err(OpError::IllegalStateTransition) } } } @@ -128,21 +160,26 @@ pub(crate) async fn join_ring( where CB: ConnectionBridge, { + let sender; + let tx = *join_op.id(); let result = if let Some(state) = op_storage.pop_join_ring_op(join_op.id()) { + sender = join_op.sender().cloned(); // was an existing operation, the other peer messaged back - update_state(state, join_op) + update_state(conn_manager, state, join_op, &op_storage.ring).await } else { + sender = join_op.sender().cloned(); // new request to join from this node, initialize the machine let machine = JoinRingOp(StateMachine::new()); - update_state(machine, join_op) + update_state(conn_manager, machine, join_op, &op_storage.ring).await }; - match result.map_err(|_| OpExecutionError::UpdateFailure) { - Err(tx) => { - log::error!("error while processing join request: {}", tx); - let tx = todo!(); - conn_manager.send(Message::Canceled(tx)).await?; - return Err(OpExecutionError::UpdateFailure.into()); + match result { + Err(err) => { + log::error!("error while processing join request: {}", err); + if let Some(sender) = sender { + conn_manager.send(&sender, Message::Canceled(tx)).await?; + } + return Err(err); } Ok(OperationResult { return_msg: Some(msg), @@ -150,7 +187,9 @@ where }) => { // updated op let id = *msg.id(); - conn_manager.send(msg).await?; + if let Some(target) = msg.sender().cloned() { + conn_manager.send(&target, msg).await?; + } op_storage.push_join_ring_op(id, updated_state)?; } Ok(OperationResult { @@ -158,7 +197,9 @@ where state: None, }) => { // finished the operation at this node, informing back - conn_manager.send(msg).await?; + if let Some(target) = msg.sender().cloned() { + conn_manager.send(&target, msg).await?; + } } Ok(OperationResult { return_msg: None, @@ -172,50 +213,206 @@ where } #[inline(always)] -fn update_state( - current_state: JoinRingOp, +async fn update_state( + conn_manager: &mut CB, + state: JoinRingOp, other_host_msg: JoinRingMsg, -) -> Result, ()> { - // let join_response_cb = - // move |sender: PeerKeyLocation, join_res: Message| -> conn_manager::Result<()> { - // let (accepted_by, tx) = if let Message::JoinResponse( - // incoming_tx, - // messages::JoinResponse::Initial { - // accepted_by, - // your_location, - // .. - // }, - // ) = join_res - // { - // log::debug!("JoinResponse received from {}", sender.peer,); - // if incoming_tx != tx { - // return Err(conn_manager::ConnError::UnexpectedTx(tx, incoming_tx)); - // } - // let loc = &mut *ring_proto.location.write(); - // *loc = Some(your_location); - // (accepted_by, incoming_tx) - // } else { - // return Err(conn_manager::ConnError::UnexpectedResponseMessage(join_res)); - // }; - // let self_location = &*ring_proto.location.read(); - // let self_location = - // &self_location.ok_or(conn_manager::ConnError::LocationUnknown)?; - // for new_peer_key in accepted_by { - // if ring_proto.ring.should_accept( - // self_location, - // &new_peer_key - // .location - // .ok_or(conn_manager::ConnError::LocationUnknown)?, - // ) { - // log::info!("Establishing connection to {}", new_peer_key.peer); - // ring_proto.establish_conn(new_peer_key, tx); - // } else { - // log::debug!("Not accepting connection to {}", new_peer_key.peer); - // } - // } - // Ok(()) - // }; - todo!() + ring: &Ring, +) -> Result, OpError> +where + CB: ConnectionBridge, +{ + let return_msg; + let new_state; + match other_host_msg { + JoinRingMsg::Req { + id, + msg: + JoinRequest::Initial { + target_loc: your_location, + req_peer, + hops_to_live, + max_hops_to_live, + }, + } => { + log::debug!( + "Initial join request received by {} with HTL {}", + req_peer, + hops_to_live + ); + + let new_location = Location::random(); + let accepted_by = if ring.should_accept( + &your_location + .location + .ok_or(OpExecutionError::TxUpdateFailure(id))?, + &new_location, + ) { + log::debug!( + "Accepting connections from {}, establising connection @ {}", + req_peer, + &your_location.peer + ); + // FIXME: self_cp.establish_conn(peer_key_loc, tx); + vec![your_location] + } else { + log::debug!("Not accepting new connection for sender {}", req_peer); + Vec::new() + }; + + log::debug!( + "Sending JoinResponse to {} accepting {} connections", + req_peer, + accepted_by.len() + ); + + let join_response = Message::from(JoinRingMsg::Resp { + id, + sender: your_location, + msg: JoinResponse::Initial { + accepted_by: accepted_by.clone(), + your_location: new_location, + your_peer_id: req_peer, + }, + }); + let new_peer_loc = PeerKeyLocation { + location: Some(new_location), + peer: req_peer, + }; + + if hops_to_live > 0 && !ring.connections_by_location.read().is_empty() { + let forward_to = if hops_to_live >= ring.rnd_if_htl_above { + log::debug!( + "Randomly selecting peer to forward JoinRequest, sender: {}", + req_peer + ); + ring.random_peer(|p| p.peer != req_peer) + } else { + log::debug!( + "Selecting close peer to forward request, sender: {}", + req_peer + ); + ring.connections_by_location + .read() + .get(&new_location) + .filter(|it| it.peer != req_peer) + .copied() + }; + + if let Some(forward_to) = forward_to { + let forwarded = Message::from(JoinRingMsg::Req { + id, + msg: JoinRequest::Proxy { + joiner: new_peer_loc, + hops_to_live: hops_to_live.min(ring.max_hops_to_live) - 1, + }, + }); + log::debug!( + "Forwarding JoinRequest from sender {} to {}", + req_peer, + forward_to.peer + ); + conn_manager.send(&forward_to, forwarded).await?; + let _forwarded_acceptors = accepted_by.into_iter().collect::>(); + // this will would jump to JoinRingMsg::Resp::JoinResponse::Proxy after peer return + // TODO: add a new state that transits from Connecting -> WaitingProxyResponse + todo!() + } else { + new_state = Some(state); + return_msg = Some(join_response); + } + } else { + new_state = Some(state); + return_msg = Some(join_response); + } + } + JoinRingMsg::Req { + id, + msg: + JoinRequest::Proxy { + joiner, + hops_to_live, + }, + } => { + todo!() + } + JoinRingMsg::Resp { + id, + sender, + msg: + JoinResponse::Initial { + accepted_by, + your_location, + your_peer_id, + }, + } => { + log::debug!("JoinResponse received from {}", sender.peer,); + // state.0.consume(input); + + // let loc = &mut *ring.location.write(); + // *loc = Some(your_location); + // let self_location = &*ring_proto.location.read(); + // let self_location = &self_location.ok_or(conn_manager::ConnError::LocationUnknown)?; + // for new_peer_key in accepted_by { + // if ring_proto.ring.should_accept( + // self_location, + // &new_peer_key + // .location + // .ok_or(conn_manager::ConnError::LocationUnknown)?, + // ) { + // log::info!("Establishing connection to {}", new_peer_key.peer); + // ring_proto.establish_conn(new_peer_key, tx); + // } else { + // log::debug!("Not accepting connection to {}", new_peer_key.peer); + // } + // } + todo!() + } + JoinRingMsg::Resp { + id, + sender, + msg: JoinResponse::Proxy { accepted_by }, + } => { + // let register_acceptors = + // move |jr_sender: PeerKeyLocation, join_resp| -> conn_manager::Result<()> { + // if let Message::JoinResponse(tx, resp) = join_resp { + // let new_acceptors = match resp { + // JoinResponse::Initial { accepted_by, .. } => accepted_by, + // JoinResponse::Proxy { accepted_by, .. } => accepted_by, + // }; + // let fa = &mut *forwarded_acceptors.lock(); + // new_acceptors.iter().for_each(|p| { + // if !fa.contains(p) { + // fa.insert(*p); + // } + // }); + // let msg = Message::from(( + // tx, + // JoinResponse::Proxy { + // accepted_by: new_acceptors, + // }, + // )); + // self_cp2.conn_manager.send(jr_sender, tx, msg)?; + // }; + // Ok(()) + // }; + todo!() + } + JoinRingMsg::Resp { + id, + sender, + msg: JoinResponse::ReceivedOC { .. }, + } => { + // + todo!() + } + JoinRingMsg::Connected => todo!(), + } + + Ok(OperationResult { + return_msg, + state: new_state, + }) } pub(crate) async fn initial_join_request( @@ -245,8 +442,10 @@ where let join_req = Message::from(messages::JoinRingMsg::Req { id: tx, msg: messages::JoinRequest::Initial { - key: this_peer.peer, + target_loc: gateway, + req_peer: this_peer, hops_to_live: max_hops_to_live, + max_hops_to_live, }, }); log::debug!( @@ -254,46 +453,44 @@ where join_req, gateway.peer ); - conn_manager.send(join_req).await?; - join_op - .0 - .consume(&JRInput::Connecting { gateway, this_peer }) - .map_err(|_| OpError::IllegalStateTransition)?; + conn_manager.send(&gateway, join_req).await?; + // join_op + // .0 + // .consume(&JoinRingMsg::Req { , this_peer }) + // .map_err(|_| OpError::IllegalStateTransition)?; op_storage.push_join_ring_op(tx, join_op)?; Ok(()) } -// fn establish_conn(self: &Arc, new_peer: PeerKeyLocation, tx: Transaction) { -// self.conn_manager.add_connection(new_peer, false); -// let self_cp = self.clone(); +// fn establish_conn(conn_manager: &mut CB, new_peer: PeerKeyLocation, tx: Transaction) +// where +// CB: ConnectionBridge, +// { +// conn_manager.add_connection(new_peer, false); // let state = Arc::new(RwLock::new(messages::OpenConnection::Connecting)); -// let state_cp = state.clone(); // let ack_peer = move |peer: PeerKeyLocation, msg: Message| -> conn_manager::Result<()> { // let (tx, oc) = match msg { // Message::OpenConnection(tx, oc) => (tx, oc), // msg => return Err(conn_manager::ConnError::UnexpectedResponseMessage(msg)), // }; -// let mut current_state = state_cp.write(); // current_state.transition(oc); // if !current_state.is_connected() { // let open_conn: Message = (tx, *current_state).into(); // log::debug!("Acknowledging OC"); -// self_cp -// .conn_manager -// .send(peer, *open_conn.id(), open_conn)?; +// conn_manager.send(peer, *open_conn.id(), open_conn)?; // } else { // log::info!( // "{} connected to {}, adding to ring", -// self_cp.peer_key, +// peer_key, // new_peer.peer // ); -// self_cp.conn_manager.send( +// conn_manager.send( // peer, // tx, // Message::from((tx, messages::OpenConnection::Connected)), // )?; -// self_cp.ring.connections_by_location.write().insert( +// ring.connections_by_location.write().insert( // new_peer // .location // .ok_or(conn_manager::ConnError::LocationUnknown)?, @@ -329,184 +526,24 @@ where // }); // } -// fn listen_for_join_req(self: &Arc) -> ListenerHandle { -// let self_cp = self.clone(); -// let process_join_req = move |sender: PeerKeyLocation, -// msg: Message| -// -> conn_manager::Result<()> { -// let (tx, join_req) = if let Message::JoinRequest(id, join_req) = msg { -// (id, join_req) -// } else { -// return Err(conn_manager::ConnError::UnexpectedResponseMessage(msg)); -// }; -// enum ReqType { -// Initial, -// Proxy, -// } -// let peer_key_loc; -// let req_type; -// let jr_hpt = match join_req { -// messages::JoinRequest::Initial { key, hops_to_live } => { -// peer_key_loc = PeerKeyLocation { -// peer: key, -// location: Some(Location::random()), -// }; -// req_type = ReqType::Initial; -// hops_to_live -// } -// messages::JoinRequest::Proxy { -// joiner, -// hops_to_live, -// } => { -// peer_key_loc = joiner; -// req_type = ReqType::Proxy; -// hops_to_live -// } -// }; -// log::debug!( -// "JoinRequest received by {} with HTL {}", -// sender -// .location -// .ok_or(conn_manager::ConnError::LocationUnknown)?, -// jr_hpt -// ); -// let your_location = self_cp -// .location -// .read() -// .ok_or(conn_manager::ConnError::LocationUnknown)?; -// let accepted_by = if self_cp.ring.should_accept( -// &your_location, -// &peer_key_loc -// .location -// .ok_or(conn_manager::ConnError::LocationUnknown)?, -// ) { -// log::debug!( -// "Accepting connections to {:?}, establising connection @ {}", -// peer_key_loc, -// self_cp.peer_key -// ); -// self_cp.establish_conn(peer_key_loc, tx); -// vec![PeerKeyLocation { -// peer: self_cp.peer_key, -// location: Some(your_location), -// }] -// } else { -// log::debug!("Not accepting new connection sender {:?}", peer_key_loc); -// Vec::new() -// }; -// log::debug!( -// "Sending JoinResponse to {} accepting {} connections", -// sender.peer, -// accepted_by.len() -// ); -// let join_response = match req_type { -// ReqType::Initial => Message::from(( -// tx, -// JoinResponse::Initial { -// accepted_by: accepted_by.clone(), -// your_location: peer_key_loc -// .location -// .ok_or(conn_manager::ConnError::LocationUnknown)?, -// your_peer_id: peer_key_loc.peer, -// }, -// )), -// ReqType::Proxy => Message::from(( -// tx, -// JoinResponse::Proxy { -// accepted_by: accepted_by.clone(), -// }, -// )), -// }; -// self_cp.conn_manager.send(peer_key_loc, tx, join_response)?; -// // NOTE: this is in practica a jump to: join_ring.join_response_cb -// if jr_hpt > 0 && !self_cp.ring.connections_by_location.read().is_empty() { -// let forward_to = if jr_hpt >= self_cp.rnd_if_htl_above { -// log::debug!( -// "Randomly selecting peer to forward JoinRequest sender {}", -// sender.peer -// ); -// self_cp.ring.random_peer(|p| p.peer != sender.peer) -// } else { -// log::debug!( -// "Selecting close peer to forward request sender {}", -// sender.peer -// ); -// self_cp -// .ring -// .connections_by_location -// .read() -// .get( -// &peer_key_loc -// .location -// .ok_or(conn_manager::ConnError::LocationUnknown)?, -// ) -// .filter(|it| it.peer != sender.peer) -// .copied() -// }; -// if let Some(forward_to) = forward_to { -// let forwarded = Message::from(( -// tx, -// JoinRequest::Proxy { -// joiner: peer_key_loc, -// hops_to_live: jr_hpt.min(self_cp.max_hops_to_live) - 1, -// }, -// )); -// let forwarded_acceptors = -// Arc::new(Mutex::new(accepted_by.into_iter().collect::>())); -// log::debug!( -// "Forwarding JoinRequest sender {} to {}", -// sender.peer, -// forward_to.peer -// ); -// let self_cp2 = self_cp.clone(); -// let register_acceptors = -// move |jr_sender: PeerKeyLocation, join_resp| -> conn_manager::Result<()> { -// if let Message::JoinResponse(tx, resp) = join_resp { -// let new_acceptors = match resp { -// JoinResponse::Initial { accepted_by, .. } => accepted_by, -// JoinResponse::Proxy { accepted_by, .. } => accepted_by, -// }; -// let fa = &mut *forwarded_acceptors.lock(); -// new_acceptors.iter().for_each(|p| { -// if !fa.contains(p) { -// fa.insert(*p); -// } -// }); -// let msg = Message::from(( -// tx, -// JoinResponse::Proxy { -// accepted_by: new_acceptors, -// }, -// )); -// self_cp2.conn_manager.send(jr_sender, tx, msg)?; -// }; -// Ok(()) -// }; -// self_cp.conn_manager.send_with_callback( -// forward_to, -// tx, -// forwarded, -// register_acceptors, -// )?; -// } -// } -// Ok(()) -// }; -// self.conn_manager -// .listen(::msg_type_id(), process_join_req) -// } - mod messages { use super::*; - use crate::{conn_manager::PeerKeyLocation, ring_proto::Location, PeerKey}; + use crate::{conn_manager::PeerKeyLocation, ring::Location, PeerKey}; use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize, Clone)] pub(crate) enum JoinRingMsg { - Req { id: Transaction, msg: JoinRequest }, - OC { id: Transaction, msg: JoinResponse }, - Resp { id: Transaction, msg: JoinResponse }, + Req { + id: Transaction, + msg: JoinRequest, + }, + Resp { + id: Transaction, + sender: PeerKeyLocation, + msg: JoinResponse, + }, + Connected, } impl JoinRingMsg { @@ -514,8 +551,17 @@ mod messages { use JoinRingMsg::*; match self { Req { id, .. } => id, - OC { id, .. } => id, Resp { id, .. } => id, + Connected => todo!(), + } + } + + pub fn sender(&self) -> Option<&PeerKeyLocation> { + use JoinRingMsg::*; + match self { + Req { .. } => None, + Resp { sender, .. } => Some(sender), + Connected => todo!(), } } } @@ -523,8 +569,10 @@ mod messages { #[derive(Debug, Serialize, Deserialize, Clone)] pub(crate) enum JoinRequest { Initial { - key: PeerKey, + target_loc: PeerKeyLocation, + req_peer: PeerKey, hops_to_live: usize, + max_hops_to_live: usize, }, Proxy { joiner: PeerKeyLocation, @@ -539,6 +587,9 @@ mod messages { your_location: Location, your_peer_id: PeerKey, }, + ReceivedOC { + by_peer: PeerKeyLocation, + }, Proxy { accepted_by: Vec, }, @@ -550,10 +601,11 @@ mod tests { use libp2p::identity::Keypair; use super::*; - use crate::PeerKey; + use crate::{message::TransactionTypeId, PeerKey}; #[test] fn join_ring_transitions() { + let id = Transaction::new(TransactionTypeId::JoinRing); let h1 = PeerKeyLocation { peer: PeerKey::from(Keypair::generate_ed25519().public()), location: None, @@ -565,25 +617,30 @@ mod tests { let mut join_op_host_1 = StateMachine::::new(); let res = join_op_host_1 - .consume(&JRInput::Connecting { - gateway: h2, - this_peer: h1, + .consume(&JoinRingMsg::Req { + id, + msg: JoinRequest::Initial { + target_loc: h1, + req_peer: h2.peer, + hops_to_live: 0, + max_hops_to_live: 0, + }, }) .unwrap() .unwrap(); - assert!(matches!(res, JROutput::OCReceived { by_peer: h1 })); - - let mut join_op_host_2 = StateMachine::::new(); - let res = join_op_host_2 - .consume(&JRInput::OCReceived) - .unwrap() - .unwrap(); - assert!(matches!(res, JROutput::Connected)); - - let res = join_op_host_1 - .consume(&JRInput::Connected) - .unwrap() - .unwrap(); - assert!(matches!(res, JROutput::Connected)); + // assert!(matches!(res, JROutput::OCReceived { by_peer: h1 })); + + // let mut join_op_host_2 = StateMachine::::new(); + // let res = join_op_host_2 + // .consume(&JRInput::OCReceived) + // .unwrap() + // .unwrap(); + // assert!(matches!(res, JROutput::Connected)); + + // let res = join_op_host_1 + // .consume(&JRInput::Connected) + // .unwrap() + // .unwrap(); + // assert!(matches!(res, JROutput::Connected)); } } diff --git a/crates/freenet2-node/src/ring_proto.rs b/crates/freenet2-node/src/ring.rs similarity index 78% rename from crates/freenet2-node/src/ring_proto.rs rename to crates/freenet2-node/src/ring.rs index db911ef92..70a75c363 100644 --- a/crates/freenet2-node/src/ring_proto.rs +++ b/crates/freenet2-node/src/ring.rs @@ -9,19 +9,40 @@ use crate::conn_manager::{self, PeerKeyLocation}; #[derive(Debug)] pub(crate) struct Ring { pub connections_by_location: RwLock>, + pub rnd_if_htl_above: usize, + pub max_hops_to_live: usize, } impl Ring { const MIN_CONNECTIONS: usize = 10; const MAX_CONNECTIONS: usize = 20; - fn new() -> Self { + /// Above this number of remaining hops, + /// randomize which of node a message which be forwarded to. + pub const RAND_WALK_ABOVE_HTL: usize = 7; + + /// + pub const MAX_HOPS_TO_LIVE: usize = 10; + + pub fn new() -> Self { Ring { connections_by_location: RwLock::new(BTreeMap::new()), + rnd_if_htl_above: Self::RAND_WALK_ABOVE_HTL, + max_hops_to_live: Self::MAX_HOPS_TO_LIVE, } } - fn should_accept(&self, my_location: &Location, location: &Location) -> bool { + pub fn with_rnd_walk_above(&mut self, rnd_if_htl_above: usize) -> &mut Self { + self.rnd_if_htl_above = rnd_if_htl_above; + self + } + + pub fn with_max_hops(&mut self, max_hops_to_live: usize) -> &mut Self { + self.max_hops_to_live = max_hops_to_live; + self + } + + pub fn should_accept(&self, my_location: &Location, location: &Location) -> bool { let cbl = &*self.connections_by_location.read(); if location == my_location || cbl.contains_key(location) { false @@ -34,7 +55,7 @@ impl Ring { } } - fn median_distance_to(&self, location: &Location) -> Distance { + pub fn median_distance_to(&self, location: &Location) -> Distance { let mut conn_by_dist = self.connections_by_distance(location); conn_by_dist.sort_by_key(|(k, _)| *k); let idx = self.connections_by_location.read().len() / 2; @@ -49,7 +70,7 @@ impl Ring { .collect() } - fn random_peer(&self, filter_fn: F) -> Option + pub fn random_peer(&self, filter_fn: F) -> Option where F: FnMut(&&PeerKeyLocation) -> bool, { @@ -142,5 +163,5 @@ pub(crate) enum RingProtoError { #[error("failed while attempting to join a ring")] Join, #[error(transparent)] - ConnError(#[from] conn_manager::ConnError), + ConnError(#[from] Box), } From 50d1889c46d540a7442ad6a2c1b7da1f0f74222f Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Thu, 23 Sep 2021 06:31:40 +0200 Subject: [PATCH 05/14] Fix state transitions w/ msg input --- .../freenet2-node/src/operations/join_ring.rs | 97 +++++++++---------- 1 file changed, 48 insertions(+), 49 deletions(-) diff --git a/crates/freenet2-node/src/operations/join_ring.rs b/crates/freenet2-node/src/operations/join_ring.rs index 750a5f578..6eb1d0cd1 100644 --- a/crates/freenet2-node/src/operations/join_ring.rs +++ b/crates/freenet2-node/src/operations/join_ring.rs @@ -39,7 +39,6 @@ impl StateMachineImpl for InternalJROp { ( JRState::Initializing, JoinRingMsg::Req { - id, msg: JoinRequest::Initial { req_peer, @@ -47,18 +46,24 @@ impl StateMachineImpl for InternalJROp { max_hops_to_live, .. }, + .. }, ) => Some(JRState::Connecting(ConnectionInfo { gateway: *target_loc, this_peer: *req_peer, max_hops_to_live: *max_hops_to_live, })), - // (JRState::Connecting { .. }, JRInput::Connecting { .. }) => Some(JRState::OCReceived), ( - JRState::Connecting { .. }, + JRState::Connecting { .. } | JRState::Initializing, + JoinRingMsg::Resp { + msg: JoinResponse::ReceivedOC { .. }, + .. + }, + ) => Some(JRState::OCReceived), + ( + JRState::Connecting { .. } | JRState::OCReceived, JoinRingMsg::Req { .. } | JoinRingMsg::Connected { .. }, ) => Some(JRState::Connected), - (JRState::OCReceived { .. }, JoinRingMsg::Connected) => Some(JRState::Connected), (JRState::Connected, _) => None, _ => None, } @@ -88,21 +93,14 @@ impl StateMachineImpl for InternalJROp { }, }), ( - JRState::Connecting { .. }, + JRState::Initializing | JRState::Connecting(_), JoinRingMsg::Resp { - id, - sender, - msg: JoinResponse::ReceivedOC { by_peer }, - }, - ) => Some(JoinRingMsg::Resp { - id: *id, - msg: JoinResponse::ReceivedOC { by_peer: *by_peer }, - sender: *sender, - }), - // (JRState::Connecting { .. }, JRInput::OCReceived | JRInput::Connected) => { - // Some(JROutput::Connected) - // } - // (JRState::OCReceived, &JoinRingMsg::OC) => Some(JROutput::Connected), + msg: JoinResponse::ReceivedOC { .. }, + .. + } + | JoinRingMsg::Connected, + ) => Some(JoinRingMsg::Connected), + (JRState::OCReceived, JoinRingMsg::Connected) => Some(JoinRingMsg::Connected), _ => None, } } @@ -133,20 +131,6 @@ impl JRState { } } -enum JRInput { - Connecting { - gateway: PeerKeyLocation, - this_peer: PeerKeyLocation, - }, - OCReceived, - Connected, -} - -enum JROutput { - OCReceived { by_peer: PeerKeyLocation }, - Connected, -} - /// Join ring routine, called upon processing a request to join or performing /// a join operation for this node. /// @@ -532,7 +516,7 @@ mod messages { use serde::{Deserialize, Serialize}; - #[derive(Debug, Serialize, Deserialize, Clone)] + #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] pub(crate) enum JoinRingMsg { Req { id: Transaction, @@ -566,7 +550,7 @@ mod messages { } } - #[derive(Debug, Serialize, Deserialize, Clone)] + #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] pub(crate) enum JoinRequest { Initial { target_loc: PeerKeyLocation, @@ -580,7 +564,7 @@ mod messages { }, } - #[derive(Debug, Serialize, Deserialize, Clone)] + #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] pub(crate) enum JoinResponse { Initial { accepted_by: Vec, @@ -628,19 +612,34 @@ mod tests { }) .unwrap() .unwrap(); - // assert!(matches!(res, JROutput::OCReceived { by_peer: h1 })); - - // let mut join_op_host_2 = StateMachine::::new(); - // let res = join_op_host_2 - // .consume(&JRInput::OCReceived) - // .unwrap() - // .unwrap(); - // assert!(matches!(res, JROutput::Connected)); - - // let res = join_op_host_1 - // .consume(&JRInput::Connected) - // .unwrap() - // .unwrap(); - // assert!(matches!(res, JROutput::Connected)); + let expected = JoinRingMsg::Resp { + id, + sender: h2, + msg: JoinResponse::ReceivedOC { by_peer: h1 }, + }; + assert_eq!(res, expected); + assert!(matches!(join_op_host_1.state(), JRState::Connecting(_))); + + let mut join_op_host_2 = StateMachine::::new(); + let res = join_op_host_2.consume(&res).unwrap().unwrap(); + let expected = JoinRingMsg::Connected; + assert_eq!(res, expected); + assert!(matches!(join_op_host_2.state(), JRState::OCReceived)); + + let res = join_op_host_1.consume(&res).unwrap().unwrap(); + let expected = JoinRingMsg::Connected; + assert_eq!(res, expected); + assert!(matches!(join_op_host_1.state(), JRState::Connected)); + + let res = join_op_host_2.consume(&res).unwrap().unwrap(); + let expected = JoinRingMsg::Connected; + assert_eq!(res, expected); + assert!(matches!(join_op_host_2.state(), JRState::Connected)); + + // transaction finished, should not return anymore + assert!(join_op_host_1.consume(&res).is_err()); + assert!(join_op_host_2.consume(&res).is_err()); + assert!(matches!(join_op_host_1.state(), JRState::Connected)); + assert!(matches!(join_op_host_2.state(), JRState::Connected)); } } From d4d279ab9d5b6d3e313acce9c302c6f7c3c82c14 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Thu, 23 Sep 2021 19:18:05 +0200 Subject: [PATCH 06/14] Adapt mem conn manager --- .../src/conn_manager/in_memory.rs | 182 +++--------------- 1 file changed, 31 insertions(+), 151 deletions(-) diff --git a/crates/freenet2-node/src/conn_manager/in_memory.rs b/crates/freenet2-node/src/conn_manager/in_memory.rs index c77880535..68aded1ef 100644 --- a/crates/freenet2-node/src/conn_manager/in_memory.rs +++ b/crates/freenet2-node/src/conn_manager/in_memory.rs @@ -1,53 +1,34 @@ //! A in-memory connection manager and transport implementation. Used for testing pourpouses. -use std::{array::IntoIter, collections::HashMap, io::Cursor, sync::Arc, time::Duration}; +use std::{io::Cursor, sync::Arc, time::Duration}; use crossbeam::channel::{self, Receiver, Sender}; use once_cell::sync::OnceCell; -use parking_lot::{Mutex, RwLock}; +use parking_lot::Mutex; use super::{ConnError, Transport}; use crate::{ config::tracing::Logger, - conn_manager::{self, ConnectionBridge, ListenerHandle, PeerKey, PeerKeyLocation}, - message::{Message, Transaction, TransactionTypeId}, + conn_manager::{ConnectionBridge, PeerKey, PeerKeyLocation}, + message::Message, ring::Location, }; - -type InboundListenerFn = - Box conn_manager::Result<()> + Send + Sync>; -type InboundListenerRegistry = RwLock>; - -type ResponseListenerFn = - Box conn_manager::Result<()> + Send + Sync>; -type OutboundListenerRegistry = Arc>>; +static NETWORK_WIRES: OnceCell<(Sender, Receiver)> = + OnceCell::new(); #[derive(Clone)] pub(crate) struct MemoryConnManager { - /// listeners for inbound initial messages - inbound_listeners: Arc>, - /// listeners for outbound messages replies - outbound_listeners: OutboundListenerRegistry, transport: InMemoryTransport, - // LIFO stack for pending listeners - pend_listeners: Sender<(Transaction, ResponseListenerFn)>, + msg_queue: Arc>>, } impl MemoryConnManager { pub fn new(is_open: bool, peer: PeerKey, location: Option) -> Self { Logger::init_logger(); - let (pend_listeners, rcv_pend_listeners) = channel::unbounded(); let transport = InMemoryTransport::new(is_open, peer, location); - let inbound_listeners: Arc> = Arc::new( - IntoIter::new(TransactionTypeId::enumeration()) - .map(|id| (id, RwLock::new(HashMap::new()))) - .collect(), - ); - let outbound_listeners: Arc>> = - Arc::new(RwLock::new(HashMap::new())); + let msg_queue = Arc::new(Mutex::new(Vec::new())); + let msg_queue_cp = msg_queue.clone(); let tr_cp = transport.clone(); - let inbound_cp = Arc::clone(&inbound_listeners); - let outbound_cp = outbound_listeners.clone(); tokio::spawn(async move { // evaluate the messages as they arrive loop { @@ -55,52 +36,18 @@ impl MemoryConnManager { if let Some(msg) = msg { let msg_data: Message = bincode::deserialize_from(Cursor::new(msg.data)).unwrap(); - if let Some(tx_fn) = outbound_cp.read().get(msg_data.id()) { - log::debug!("Received response for transaction: {}", msg_data.id()); - if let Some(location) = msg.origin_loc { - if let Err(err) = tx_fn( - PeerKeyLocation { - peer: msg.origin, - location: Some(location), - }, - msg_data, - ) { - log::error!("Error processing response: {}", err); - } - } else { - log::error!("No location for responding peer {}", msg.target); - } - } else { - let listeners = &inbound_cp[&msg_data.msg_type()]; - log::debug!("Received inbound transaction: {}", msg_data.id()); - let reg = &*listeners.read(); - for func in reg.values() { - if let Err(err) = func( - PeerKeyLocation { - peer: msg.origin, - location: None, - }, - msg_data.clone(), - ) { - log::error!("Error while calling inbound msg handler: {}", err); - } - } - } - // insert any pending functions generated from within the callback - let mut lock = outbound_cp.write(); - for (tx, func) in rcv_pend_listeners.try_iter() { - lock.insert(tx, func); + if let Some(mut queue) = msg_queue_cp.try_lock() { + queue.push(msg_data); + std::mem::drop(queue); } } - tokio::time::sleep(Duration::from_millis(1)).await; + tokio::time::sleep(Duration::from_nanos(1000)).await; } }); Self { - inbound_listeners, - outbound_listeners, transport, - pend_listeners, + msg_queue, } } } @@ -108,97 +55,30 @@ impl MemoryConnManager { #[async_trait::async_trait] impl ConnectionBridge for MemoryConnManager { async fn recv(&self) -> Result { - todo!() + loop { + if let Some(mut queue) = self.msg_queue.try_lock() { + if let Some(msg) = queue.pop() { + return Ok(msg); + } + std::mem::drop(queue); + } + tokio::time::sleep(Duration::from_nanos(1000)).await; + } } async fn send(&self, target: &PeerKeyLocation, msg: Message) -> Result<(), ConnError> { - todo!() + let msg = bincode::serialize(&msg)?; + self.transport.send( + target.peer, + target.location.ok_or(ConnError::LocationUnknown)?, + msg, + ); + Ok(()) } - fn add_connection(&mut self, peer: PeerKeyLocation, unsolicited: bool) { - todo!() - } + fn add_connection(&mut self, _peer: PeerKeyLocation, _unsolicited: bool) {} } -// impl ConnectionBridge for MemoryConnManager { -// type Transport = InMemoryTransport; - -// fn listen(&self, tx_type: TransactionTypeId, listen_fn: F) -> ListenerHandle -// where -// F: Fn(PeerKeyLocation, Message) -> conn_manager::Result<()> + Send + Sync + 'static, -// { -// let tx_ty_listener = &self.inbound_listeners[&tx_type]; -// let handle_id = ListenerHandle::new(); -// tx_ty_listener -// .write() -// .insert(handle_id, Box::new(listen_fn)); -// handle_id -// } - -// fn listen_to_replies(&self, tx: Transaction, callback: F) -// where -// F: Fn(PeerKeyLocation, Message) -> conn_manager::Result<()> + Send + Sync + 'static, -// { -// // optimistically try to acquire a lock -// if let Some(mut lock) = self.outbound_listeners.try_write() { -// lock.insert(tx, Box::new(callback)); -// } else { -// // it failed, this is being inserted from an other existing closure holding the lock -// // send it to the temporal stack queue for posterior insertion -// self.pend_listeners -// .send((tx, Box::new(callback))) -// .expect("full or disconnected"); -// } -// } - -// fn transport(&self) -> &Self::Transport { -// &self.transport -// } - -// fn add_connection(&self, _peer_key: PeerKeyLocation, _unsolicited: bool) {} - -// fn send_with_callback( -// &self, -// to: PeerKeyLocation, -// tx: Transaction, -// msg: Message, -// callback: F, -// ) -> conn_manager::Result<()> -// where -// F: Fn(PeerKeyLocation, Message) -> conn_manager::Result<()> + Send + Sync + 'static, -// { -// // store listening func -// self.outbound_listeners -// .write() -// .insert(tx, Box::new(callback)); - -// // send the msg -// let serialized = bincode::serialize(&msg)?; -// self.transport -// .send(to.peer, to.location.unwrap(), serialized); -// Ok(()) -// } - -// fn send( -// &self, -// to: PeerKeyLocation, -// _tx: Transaction, -// msg: Message, -// ) -> conn_manager::Result<()> { -// let serialized = bincode::serialize(&msg)?; -// self.transport -// .send(to.peer, to.location.unwrap(), serialized); -// Ok(()) -// } - -// fn remove_listener(&self, tx: Transaction) { -// self.outbound_listeners.write().remove(&tx); -// } -// } - -static NETWORK_WIRES: OnceCell<(Sender, Receiver)> = - OnceCell::new(); - #[derive(Clone, Debug)] struct MessageOnTransit { origin: PeerKey, From 5932ed54d72afd76f927686f0a5d9f02a524f6bf Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Fri, 24 Sep 2021 21:17:49 +0200 Subject: [PATCH 07/14] Rewrite test infra --- crates/freenet2-node/src/conn_manager.rs | 1 + .../src/conn_manager/in_memory.rs | 6 +- crates/freenet2-node/src/node.rs | 163 +++++++++++- crates/freenet2-node/src/node/in_memory.rs | 8 +- crates/freenet2-node/src/node/libp2p_impl.rs | 30 +-- crates/freenet2-node/src/node/op_state.rs | 2 +- crates/freenet2-node/src/operations.rs | 4 +- .../src/operations/_tmp_ring_proto.rs | 243 ------------------ .../freenet2-node/src/operations/join_ring.rs | 80 +++++- crates/freenet2-node/src/ring.rs | 2 +- 10 files changed, 256 insertions(+), 283 deletions(-) delete mode 100644 crates/freenet2-node/src/operations/_tmp_ring_proto.rs diff --git a/crates/freenet2-node/src/conn_manager.rs b/crates/freenet2-node/src/conn_manager.rs index 47befa80e..b5c140457 100644 --- a/crates/freenet2-node/src/conn_manager.rs +++ b/crates/freenet2-node/src/conn_manager.rs @@ -48,6 +48,7 @@ pub(crate) trait ConnectionBridge { /// A protocol used to send and receive data over the network. pub(crate) trait Transport { fn is_open(&self) -> bool; + fn location(&self) -> Option; } #[derive(Debug, PartialEq, Eq, Hash)] diff --git a/crates/freenet2-node/src/conn_manager/in_memory.rs b/crates/freenet2-node/src/conn_manager/in_memory.rs index 68aded1ef..7182e7ea3 100644 --- a/crates/freenet2-node/src/conn_manager/in_memory.rs +++ b/crates/freenet2-node/src/conn_manager/in_memory.rs @@ -17,7 +17,7 @@ static NETWORK_WIRES: OnceCell<(Sender, Receiver>>, } @@ -159,4 +159,8 @@ impl Transport for InMemoryTransport { fn is_open(&self) -> bool { self.is_open } + + fn location(&self) -> Option { + self.location + } } diff --git a/crates/freenet2-node/src/node.rs b/crates/freenet2-node/src/node.rs index 75f116f99..5515e05e3 100644 --- a/crates/freenet2-node/src/node.rs +++ b/crates/freenet2-node/src/node.rs @@ -4,7 +4,8 @@ use libp2p::{identity, multiaddr::Protocol, Multiaddr, PeerId}; use crate::config::CONF; -use self::{in_memory::InMemory, libp2p_impl::NodeLibP2P}; +use self::libp2p_impl::NodeLibP2P; +pub(crate) use in_memory::InMemory; pub(crate) use op_state::{OpExecutionError, OpStateStorage}; mod in_memory; @@ -182,3 +183,163 @@ fn multiaddr_from_connection(conn: (IpAddr, u16)) -> Multiaddr { addr.push(Protocol::Tcp(conn.1)); addr } + +#[cfg(test)] +pub mod test_utils { + use std::{ + collections::HashMap, + net::{Ipv4Addr, Ipv6Addr, SocketAddr, TcpListener}, + }; + + use libp2p::identity; + use rand::Rng; + use tokio::sync::mpsc; + + use crate::{ + conn_manager::{ConnectionBridge, Transport}, + message::Message, + node::{InMemory, InitPeerNode}, + operations::{join_ring::join_ring_op, OpError}, + ring::Distance, + NodeConfig, PeerKey, + }; + + pub fn get_free_port() -> Result { + let mut port; + for _ in 0..100 { + port = get_dynamic_port(); + let bind_addr = SocketAddr::from((Ipv4Addr::LOCALHOST, port)); + if let Ok(conn) = TcpListener::bind(bind_addr) { + std::mem::drop(conn); + return Ok(port); + } + } + Err(()) + } + + pub fn get_dynamic_port() -> u16 { + const FIRST_DYNAMIC_PORT: u16 = 49152; + const LAST_DYNAMIC_PORT: u16 = 65535; + rand::thread_rng().gen_range(FIRST_DYNAMIC_PORT..LAST_DYNAMIC_PORT) + } + + pub(crate) struct SimNetwork { + // gateways: HashMap, + // peers: HashMap, + meta_info_tx: mpsc::Sender>, + meta_info_rx: mpsc::Receiver>, + } + + pub(crate) struct NetEvent { + pub(crate) sender: String, + pub(crate) event: EventType, + } + + pub(crate) enum EventType { + /// A peer joined the network through some gateway. + JoinSuccess { gateway: PeerKey, new_node: PeerKey }, + } + + impl SimNetwork { + pub fn build( + network_size: usize, + ring_max_htl: usize, + rnd_if_htl_above: usize, + ) -> SimNetwork { + let sim = SimNetwork::new(); + + // build gateway node + // let probe_protocol = Some(ProbeProtocol::new(ring_protocol.clone(), loc)); + let gateway_pair = identity::Keypair::generate_ed25519(); + let gateway_peer_id = gateway_pair.public().into_peer_id(); + let gateway_port = get_free_port().unwrap(); + let config = NodeConfig::new() + .with_ip(Ipv6Addr::LOCALHOST) + .with_port(gateway_port) + .with_key(gateway_pair); + let gateway = InMemory::build(config).unwrap(); + sim.initialize_gateway(gateway, "gateway".to_owned()); + + // add other nodes to the simulation + for node_no in 0..network_size { + let label = format!("node-{}", node_no); + let config = NodeConfig::new().add_provider( + InitPeerNode::new() + .listening_ip(Ipv6Addr::LOCALHOST) + .listening_port(gateway_port) + .with_identifier(gateway_peer_id), + ); + sim.initialize_peer(InMemory::build(config).unwrap(), label); + } + sim + } + + pub async fn recv_net_events(&mut self) -> Option> { + self.meta_info_rx.recv().await + } + + fn new() -> Self { + let (meta_info_tx, meta_info_rx) = mpsc::channel(100); + Self { + meta_info_rx, + meta_info_tx, + } + } + + fn initialize_gateway(&self, gateway: InMemory, sender_label: String) { + let info_ch = self.meta_info_tx.clone(); + tokio::spawn(Self::listen(gateway, info_ch, sender_label)); + } + + fn initialize_peer(&self, mut peer: InMemory, sender_label: String) { + let info_ch = self.meta_info_tx.clone(); + tokio::spawn(async move { + if peer.start().await.is_err() { + let _ = info_ch.send(Err(OpError::IllegalStateTransition)).await; + return Err(()); + } + Self::listen(peer, info_ch, sender_label).await + }); + } + + async fn listen( + mut gateway: InMemory, + info_ch: mpsc::Sender>, + sender: String, + ) -> Result<(), ()> { + while let Ok(msg) = gateway.conn_manager.recv().await { + if let Message::JoinRing(msg) = msg { + if let Err(err) = + join_ring_op(&mut gateway.op_storage, &mut gateway.conn_manager, msg).await + { + let _ = info_ch.send(Err(err)).await; + return Err(()); + } + } else { + break; + } + } + Ok(()) + } + } + + /// Builds an histogram of the distribution in the ring of each node relative to each other. + fn _ring_distribution<'a>( + nodes: impl Iterator + 'a, + ) -> impl Iterator + 'a { + // TODO: groupby certain intervals + // e.g. grouping func: (it * 200.0).roundToInt().toDouble() / 200.0 + nodes + .map(|node| { + let node_ring = &node.op_storage.ring; + let self_loc = node.conn_manager.transport.location().unwrap(); + node_ring + .connections_by_location + .read() + .keys() + .map(|d| self_loc.distance(d)) + .collect::>() + }) + .flatten() + } +} diff --git a/crates/freenet2-node/src/node/in_memory.rs b/crates/freenet2-node/src/node/in_memory.rs index 5f5239bfb..4dc083155 100644 --- a/crates/freenet2-node/src/node/in_memory.rs +++ b/crates/freenet2-node/src/node/in_memory.rs @@ -7,12 +7,12 @@ use crate::{ use super::op_state::OpStateStorage; -pub(super) struct InMemory { +pub(crate) struct InMemory { peer: PeerKey, listening: bool, - conn_manager: MemoryConnManager, - op_storage: OpStateStorage, gateways: Vec, + pub conn_manager: MemoryConnManager, + pub op_storage: OpStateStorage, } impl InMemory { @@ -70,7 +70,7 @@ impl InMemory { match self.conn_manager.recv().await { Ok(msg) => match msg { Message::JoinRing(join_op) => { - join_ring::join_ring(&mut self.op_storage, &mut self.conn_manager, join_op) + join_ring::join_ring_op(&mut self.op_storage, &mut self.conn_manager, join_op) .await .unwrap(); } diff --git a/crates/freenet2-node/src/node/libp2p_impl.rs b/crates/freenet2-node/src/node/libp2p_impl.rs index a77053e39..8fd767882 100644 --- a/crates/freenet2-node/src/node/libp2p_impl.rs +++ b/crates/freenet2-node/src/node/libp2p_impl.rs @@ -155,35 +155,15 @@ impl From for NetEvent { #[cfg(test)] mod tests { - use std::{ - net::{Ipv4Addr, SocketAddr, TcpListener}, - time::Duration, - }; + use std::{net::Ipv4Addr, time::Duration}; use super::*; - use crate::{config::tracing::Logger, node::InitPeerNode}; + use crate::{ + config::tracing::Logger, + node::{test_utils::get_free_port, InitPeerNode}, + }; use libp2p::{futures::StreamExt, swarm::SwarmEvent}; - use rand::Rng; - - fn get_free_port() -> Result { - let mut port; - for _ in 0..100 { - port = get_dynamic_port(); - let bind_addr = SocketAddr::from((Ipv4Addr::LOCALHOST, port)); - if let Ok(conn) = TcpListener::bind(bind_addr) { - std::mem::drop(conn); - return Ok(port); - } - } - Err(()) - } - - fn get_dynamic_port() -> u16 { - const FIRST_DYNAMIC_PORT: u16 = 49152; - const LAST_DYNAMIC_PORT: u16 = 65535; - rand::thread_rng().gen_range(FIRST_DYNAMIC_PORT..LAST_DYNAMIC_PORT) - } /// Ping test event loop async fn ping_ev_loop(peer: &mut NodeLibP2P) -> Result<(), ()> { diff --git a/crates/freenet2-node/src/node/op_state.rs b/crates/freenet2-node/src/node/op_state.rs index 5c33c1044..49ba2bf91 100644 --- a/crates/freenet2-node/src/node/op_state.rs +++ b/crates/freenet2-node/src/node/op_state.rs @@ -39,7 +39,7 @@ impl OpStateStorage { } } -#[derive(Debug, thiserror::Error)] +#[derive(Debug, thiserror::Error, Clone)] pub(crate) enum OpExecutionError { #[error("unspected transaction type, trying to get a {0:?} from a {1:?}")] IncorrectTxType(TransactionTypeId, TransactionTypeId), diff --git a/crates/freenet2-node/src/operations.rs b/crates/freenet2-node/src/operations.rs index 30f1dbc24..baab77b60 100644 --- a/crates/freenet2-node/src/operations.rs +++ b/crates/freenet2-node/src/operations.rs @@ -53,6 +53,6 @@ macro_rules! op_type_enumeration { } op_type_enumeration!(decl struct { - join_ring: JoinRingOp, - probe_peers: ProbeOp + join_ring: JoinRingOp + // probe_peers: ProbeOp }); diff --git a/crates/freenet2-node/src/operations/_tmp_ring_proto.rs b/crates/freenet2-node/src/operations/_tmp_ring_proto.rs deleted file mode 100644 index 755655e79..000000000 --- a/crates/freenet2-node/src/operations/_tmp_ring_proto.rs +++ /dev/null @@ -1,243 +0,0 @@ -#![allow(unused)] // FIXME: remove this attr - -use std::{ - collections::{BTreeMap, HashSet}, - convert::TryFrom, - fmt::Display, - hash::Hasher, - sync::Arc, - time::{Duration, Instant}, -}; - -use parking_lot::{Mutex, RwLock}; -use serde::{Deserialize, Serialize}; - -use crate::{ - conn_manager::{self, ConnectionBridge, ListenerHandle, PeerKey, PeerKeyLocation, Transport}, - message::{Message, TransactionType, Transaction}, - ring_proto::messages::{JoinRequest, JoinResponse}, - StdResult, -}; - -type Result = StdResult; - -pub(crate) struct RingProtocol { - pub conn_manager: Arc, - peer_key: PeerKey, - /// A location gets assigned once a node joins the network via a gateway, - /// until then it has no location unless the node is a gateway. - pub location: RwLock>, - gateways: RwLock>, - max_hops_to_live: usize, - rnd_if_htl_above: usize, - pub ring: Ring, -} - -impl RingProtocol -where - T: Transport + 'static, - CM: ConnectionBridge + 'static, -{ - fn new( - conn_manager: CM, - peer_key: PeerKey, - max_hops_to_live: usize, - rnd_if_htl_above: usize, - ) -> Arc { - Arc::new(RingProtocol { - conn_manager: Arc::new(conn_manager), - peer_key, - location: RwLock::new(None), - gateways: RwLock::new(HashSet::new()), - max_hops_to_live, - rnd_if_htl_above, - ring: Ring::new(), - }) - } - - pub fn with_location(self: Arc, loc: Location) -> Arc { - *self.location.write() = Some(loc); - self - } - - fn listen_for_close_conn(&self) { - todo!() - } - -} - -#[derive(thiserror::Error, Debug)] -pub(crate) enum RingProtoError { - #[error("failed while attempting to join a ring")] - Join, - #[error(transparent)] - ConnError(#[from] conn_manager::ConnError), -} - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - - use libp2p::identity; - use rand::Rng; - - use super::{messages::OpenConnection, *}; - use crate::{ - config::tracing::Logger, - conn_manager::in_memory::MemoryConnManager, - message::ProbeRequest, - probe_proto::{self, ProbeProtocol}, - }; - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn node0_to_gateway_conn() -> StdResult<(), Box> { - //! Given a network of one node and one gateway test that both are connected. - Logger::init_logger(); - - let ring_protocols = sim_network_builder(1, 1, 0); - tokio::time::sleep(Duration::from_secs(3)).await; - - assert_eq!( - ring_protocols["node-0"] - .ring_protocol - .ring - .connections_by_location - .read() - .len(), - 1 - ); - - assert_eq!( - ring_protocols["gateway"] - .ring_protocol - .ring - .connections_by_location - .read() - .len(), - 1 - ); - - Ok(()) - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn all_nodes_should_connect() -> StdResult<(), Box> { - //! Given a network of 1000 peers all nodes should have connections. - Logger::init_logger(); - - let mut sim_nodes = sim_network_builder(10, 10, 7); - tokio::time::sleep(Duration::from_secs(300)).await; - // let _hist: Vec<_> = _ring_distribution(sim_nodes.values()).collect(); - - const NUM_PROBES: usize = 10; - let mut probe_responses = Vec::with_capacity(NUM_PROBES); - for probe_idx in 0..NUM_PROBES { - let target = Location::random(); - let idx: usize = rand::thread_rng().gen_range(0..sim_nodes.len()); - let rnd_node = sim_nodes - .get_mut(&format!("node-{}", idx)) - .ok_or("node not found")?; - let probe_response = ProbeProtocol::probe( - rnd_node.ring_protocol.clone(), - Transaction::new(::msg_type_id()), - ProbeRequest { - hops_to_live: 7, - target, - }, - ) - .await - .expect("failed to get probe response"); - probe_responses.push(probe_response); - } - // probe_proto::utils::plot_probe_responses(probe_responses); - - let any_empties = sim_nodes - .values() - .map(|node| { - node.ring_protocol - .ring - .connections_by_location - .read() - .is_empty() - }) - .any(|is_empty| is_empty); - assert!(!any_empties); - - Ok(()) - } - - struct SimulatedNode { - ring_protocol: Arc>, - probe_protocol: Option, - } - - fn sim_network_builder( - network_size: usize, - ring_max_htl: usize, - rnd_if_htl_above: usize, - // _per_node_delay: usize, - ) -> HashMap { - let mut nodes = HashMap::new(); - - // build gateway node - let keypair = identity::Keypair::generate_ed25519(); - let gw_key = keypair.public().into(); - let loc = Location::random(); - let conn_manager = MemoryConnManager::new(true, gw_key, Some(loc)); - let ring_protocol = RingProtocol::new(conn_manager, gw_key, ring_max_htl, rnd_if_htl_above) - .with_location(loc); - let probe_protocol = Some(ProbeProtocol::new(ring_protocol.clone(), loc)); - ring_protocol.listen_for_join_req(); - nodes.insert( - "gateway".to_owned(), - SimulatedNode { - ring_protocol, - probe_protocol, - }, - ); - - // add other nodes to the simulation - for node_no in 0..network_size { - let label = format!("node-{}", node_no); - let keypair = identity::Keypair::generate_ed25519(); - let peer_key = keypair.public().into(); - let conn_manager = MemoryConnManager::new(false, peer_key, None); - let ring_protocol = - RingProtocol::new(conn_manager, peer_key, ring_max_htl, rnd_if_htl_above); - ring_protocol.gateways.write().insert(PeerKeyLocation { - peer: gw_key, - location: Some(loc), - }); - ring_protocol.join_ring().unwrap(); - - nodes.insert( - label, - SimulatedNode { - ring_protocol, - probe_protocol: None, - }, - ); - } - nodes - } - - /// Builds an histogram of the distribution in the ring of each node relative to each other. - fn _ring_distribution<'a>( - nodes: impl Iterator + 'a, - ) -> impl Iterator + 'a { - // TODO: groupby certain intervals - // e.g. grouping func: (it * 200.0).roundToInt().toDouble() / 200.0 - nodes - .map(|node| { - let node_ring = &node.ring_protocol.ring; - let self_loc = node.ring_protocol.location.read().unwrap(); - node_ring - .connections_by_location - .read() - .keys() - .map(|d| self_loc.distance(d)) - .collect::>() - }) - .flatten() - } -} diff --git a/crates/freenet2-node/src/operations/join_ring.rs b/crates/freenet2-node/src/operations/join_ring.rs index 6eb1d0cd1..f64ca6f51 100644 --- a/crates/freenet2-node/src/operations/join_ring.rs +++ b/crates/freenet2-node/src/operations/join_ring.rs @@ -17,8 +17,7 @@ pub(crate) struct JoinRingOp(StateMachine); impl JoinRingOp { pub fn new(this_peer: PeerKeyLocation, gateway: PeerKeyLocation) -> Self { - let mut machine = StateMachine::new(); - JoinRingOp(machine) + JoinRingOp(StateMachine::new()) } } @@ -136,7 +135,7 @@ impl JRState { /// /// # Arguments /// - join_op: no nodes -pub(crate) async fn join_ring( +pub(crate) async fn join_ring_op( op_storage: &mut OpStateStorage, conn_manager: &mut CB, join_op: JoinRingMsg, @@ -402,7 +401,7 @@ where pub(crate) async fn initial_join_request( op_storage: &mut OpStateStorage, conn_manager: &mut CB, - mut join_op: JoinRingOp, + join_op: JoinRingOp, ) -> Result<(), OpError> where CB: ConnectionBridge, @@ -582,10 +581,17 @@ mod messages { #[cfg(test)] mod tests { + use std::time::Duration; + use libp2p::identity::Keypair; use super::*; - use crate::{message::TransactionTypeId, PeerKey}; + use crate::{ + config::tracing::Logger, + message::TransactionTypeId, + node::test_utils::{EventType, SimNetwork}, + PeerKey, + }; #[test] fn join_ring_transitions() { @@ -642,4 +648,68 @@ mod tests { assert!(matches!(join_op_host_1.state(), JRState::Connected)); assert!(matches!(join_op_host_2.state(), JRState::Connected)); } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn node0_to_gateway_conn() -> Result<(), Box> { + //! Given a network of one node and one gateway test that both are connected. + Logger::init_logger(); + let mut sim_net = SimNetwork::build(1, 1, 0); + match tokio::time::timeout(Duration::from_secs(1), sim_net.recv_net_events()).await { + Ok(Some(Ok(event))) => match event.event { + EventType::JoinSuccess { gateway, new_node } => { + log::info!("Successful join op between {} and {}", gateway, new_node) + } + }, + _ => return Err("no event received".into()), + } + Ok(()) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn all_nodes_should_connect() -> Result<(), Box> { + //! Given a network of 1000 peers all nodes should have connections. + Logger::init_logger(); + + let _sim_nodes = SimNetwork::build(10, 10, 7); + tokio::time::sleep(Duration::from_secs(300)).await; + // let _hist: Vec<_> = _ring_distribution(sim_nodes.values()).collect(); + + // FIXME: enable probing + // const NUM_PROBES: usize = 10; + // let mut probe_responses = Vec::with_capacity(NUM_PROBES); + // for probe_idx in 0..NUM_PROBES { + // let target = Location::random(); + // let idx: usize = rand::thread_rng().gen_range(0..sim_nodes.len()); + // let rnd_node = sim_nodes + // .get_mut(&format!("node-{}", idx)) + // .ok_or("node not found")?; + // let probe_response = ProbeProtocol::probe( + // rnd_node.ring_protocol.clone(), + // Transaction::new(::msg_type_id()), + // ProbeRequest { + // hops_to_live: 7, + // target, + // }, + // ) + // .await + // .expect("failed to get probe response"); + // probe_responses.push(probe_response); + // } + // probe_proto::utils::plot_probe_responses(probe_responses); + + // let any_empties = sim_nodes + // .peers + // .values() + // .map(|node| { + // node.op_storage + // .ring + // .connections_by_location + // .read() + // .is_empty() + // }) + // .any(|is_empty| is_empty); + // assert!(!any_empties); + + Ok(()) + } } diff --git a/crates/freenet2-node/src/ring.rs b/crates/freenet2-node/src/ring.rs index 70a75c363..76b4bcfe5 100644 --- a/crates/freenet2-node/src/ring.rs +++ b/crates/freenet2-node/src/ring.rs @@ -87,7 +87,7 @@ impl Ring { #[derive(Debug, serde::Serialize, serde::Deserialize, Clone, Copy)] pub struct Location(f64); -type Distance = Location; +pub(crate) type Distance = Location; impl Location { /// Returns a new random location. From 910600f07ebd6454be91bf070b2235bb8d2b4fce Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Sat, 25 Sep 2021 05:50:49 +0200 Subject: [PATCH 08/14] Add put op placeholder --- .gitignore | 2 - crates/freenet2-node/src/conn_manager.rs | 21 +--------- crates/freenet2-node/src/message.rs | 39 ++++++++----------- crates/freenet2-node/src/node.rs | 22 +++++------ crates/freenet2-node/src/node/in_memory.rs | 20 +++++----- crates/freenet2-node/src/node/op_state.rs | 20 ++++++++-- crates/freenet2-node/src/operations.rs | 9 ++++- crates/freenet2-node/src/operations/get.rs | 10 +++++ .../freenet2-node/src/operations/join_ring.rs | 8 ++-- crates/freenet2-node/src/operations/put.rs | 35 +++++++++++++++++ .../freenet2-node/src/operations/routing.rs | 2 + .../freenet2-node/src/operations/subscribe.rs | 0 12 files changed, 114 insertions(+), 74 deletions(-) create mode 100644 crates/freenet2-node/src/operations/get.rs create mode 100644 crates/freenet2-node/src/operations/put.rs create mode 100644 crates/freenet2-node/src/operations/routing.rs create mode 100644 crates/freenet2-node/src/operations/subscribe.rs diff --git a/.gitignore b/.gitignore index 7f4f4fbad..35c3e4ed1 100644 --- a/.gitignore +++ b/.gitignore @@ -18,5 +18,3 @@ target/ # IDEs .idea/* .vscode/* - -Cargo.lock \ No newline at end of file diff --git a/crates/freenet2-node/src/conn_manager.rs b/crates/freenet2-node/src/conn_manager.rs index b5c140457..d9e7bb2ed 100644 --- a/crates/freenet2-node/src/conn_manager.rs +++ b/crates/freenet2-node/src/conn_manager.rs @@ -13,29 +13,12 @@ use crate::{ pub mod in_memory; -const _PING_EVERY: Duration = Duration::from_secs(30); -const _DROP_CONN_AFTER: Duration = Duration::from_secs(30 * 10); -static HANDLE_ID: AtomicU64 = AtomicU64::new(0); +const PING_EVERY: Duration = Duration::from_secs(30); +const DROP_CONN_AFTER: Duration = Duration::from_secs(30 * 10); // pub(crate) type RemoveConnHandler<'t> = Box; pub(crate) type Result = StdResult; -/// 3 words size for 64-bit platforms. -#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] -pub(crate) struct ListenerHandle(u64); - -impl ListenerHandle { - pub fn new() -> Self { - ListenerHandle(HANDLE_ID.fetch_add(1, std::sync::atomic::Ordering::SeqCst)) - } -} - -impl Default for ListenerHandle { - fn default() -> Self { - Self::new() - } -} - #[async_trait::async_trait] pub(crate) trait ConnectionBridge { fn add_connection(&mut self, peer: PeerKeyLocation, unsolicited: bool); diff --git a/crates/freenet2-node/src/message.rs b/crates/freenet2-node/src/message.rs index 562d5f162..d12307092 100644 --- a/crates/freenet2-node/src/message.rs +++ b/crates/freenet2-node/src/message.rs @@ -3,7 +3,11 @@ use std::{fmt::Display, time::Duration}; use serde::{Deserialize, Serialize}; use uuid::Uuid; -use crate::{conn_manager::PeerKeyLocation, operations::join_ring::JoinRingMsg, ring::Location}; +use crate::{ + conn_manager::PeerKeyLocation, + operations::{join_ring::JoinRingMsg, put::PutMsg}, + ring::Location, +}; pub(crate) use sealed_msg_type::TransactionTypeId; /// An transaction is a unique, universal and efficient identifier for any @@ -48,14 +52,14 @@ impl Display for Transaction { /// Get the transaction type associated to a given message type. pub(crate) trait TransactionType: sealed_msg_type::SealedTxType { - fn msg_type_id() -> TransactionTypeId; + fn tx_type_id() -> TransactionTypeId; } impl TransactionType for T where T: sealed_msg_type::SealedTxType, { - fn msg_type_id() -> TransactionTypeId { + fn tx_type_id() -> TransactionTypeId { ::tx_type_id() } } @@ -71,13 +75,8 @@ mod sealed_msg_type { #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy, Serialize, Deserialize)] pub(crate) enum TransactionTypeId { JoinRing, - Probe, - } - - impl TransactionTypeId { - pub const fn enumeration() -> [Self; 2] { - [Self::JoinRing, Self::Probe] - } + Put, + Canceled, } macro_rules! transaction_type_enumeration { @@ -99,18 +98,18 @@ mod sealed_msg_type { } transaction_type_enumeration!(decl struct { - JoinRing -> JoinRingMsg + JoinRing -> JoinRingMsg, + Put -> PutMsg, + Canceled -> Transaction }); } #[derive(Debug, Serialize, Deserialize, Clone)] pub(crate) enum Message { JoinRing(JoinRingMsg), + Put(PutMsg), /// Failed a transaction, informing of cancellation. Canceled(Transaction), - // Probe - ProbeRequest(Transaction, ProbeRequest), - ProbeResponse(Transaction, ProbeResponse), } impl Message { @@ -122,9 +121,8 @@ impl Message { use Message::*; match self { JoinRing(op) => op.id(), - ProbeRequest(id, _) => id, - ProbeResponse(id, _) => id, - Canceled(_) => todo!(), + Put(op) => op.id(), + Canceled(tx) => tx, } } @@ -132,15 +130,10 @@ impl Message { use Message::*; match self { JoinRing(op) => op.sender(), - ProbeRequest(id, _) => None, - ProbeResponse(id, _) => None, Canceled(_) => None, + Put(op) => op.sender(), } } - - pub fn msg_type(&self) -> TransactionTypeId { - todo!() - } } impl Display for Message { diff --git a/crates/freenet2-node/src/node.rs b/crates/freenet2-node/src/node.rs index 5515e05e3..446dfd069 100644 --- a/crates/freenet2-node/src/node.rs +++ b/crates/freenet2-node/src/node.rs @@ -5,7 +5,7 @@ use libp2p::{identity, multiaddr::Protocol, Multiaddr, PeerId}; use crate::config::CONF; use self::libp2p_impl::NodeLibP2P; -pub(crate) use in_memory::InMemory; +pub(crate) use in_memory::NodeInMemory; pub(crate) use op_state::{OpExecutionError, OpStateStorage}; mod in_memory; @@ -25,7 +25,7 @@ impl Node { enum NodeImpl { LibP2P(Box), - InMemory(Box), + InMemory(Box), } pub struct NodeConfig { @@ -98,7 +98,7 @@ impl NodeConfig { /// Builds a node using in-memory transport. Used for testing pourpouses. pub fn build_in_memory(self) -> Result { - let inmem = InMemory::build(self)?; + let inmem = NodeInMemory::build(self)?; Ok(Node(NodeImpl::InMemory(Box::new(inmem)))) } } @@ -198,7 +198,7 @@ pub mod test_utils { use crate::{ conn_manager::{ConnectionBridge, Transport}, message::Message, - node::{InMemory, InitPeerNode}, + node::{InitPeerNode, NodeInMemory}, operations::{join_ring::join_ring_op, OpError}, ring::Distance, NodeConfig, PeerKey, @@ -257,7 +257,7 @@ pub mod test_utils { .with_ip(Ipv6Addr::LOCALHOST) .with_port(gateway_port) .with_key(gateway_pair); - let gateway = InMemory::build(config).unwrap(); + let gateway = NodeInMemory::build(config).unwrap(); sim.initialize_gateway(gateway, "gateway".to_owned()); // add other nodes to the simulation @@ -269,7 +269,7 @@ pub mod test_utils { .listening_port(gateway_port) .with_identifier(gateway_peer_id), ); - sim.initialize_peer(InMemory::build(config).unwrap(), label); + sim.initialize_peer(NodeInMemory::build(config).unwrap(), label); } sim } @@ -286,12 +286,12 @@ pub mod test_utils { } } - fn initialize_gateway(&self, gateway: InMemory, sender_label: String) { + fn initialize_gateway(&self, gateway: NodeInMemory, sender_label: String) { let info_ch = self.meta_info_tx.clone(); tokio::spawn(Self::listen(gateway, info_ch, sender_label)); } - fn initialize_peer(&self, mut peer: InMemory, sender_label: String) { + fn initialize_peer(&self, mut peer: NodeInMemory, sender_label: String) { let info_ch = self.meta_info_tx.clone(); tokio::spawn(async move { if peer.start().await.is_err() { @@ -303,9 +303,9 @@ pub mod test_utils { } async fn listen( - mut gateway: InMemory, + mut gateway: NodeInMemory, info_ch: mpsc::Sender>, - sender: String, + _sender: String, ) -> Result<(), ()> { while let Ok(msg) = gateway.conn_manager.recv().await { if let Message::JoinRing(msg) = msg { @@ -325,7 +325,7 @@ pub mod test_utils { /// Builds an histogram of the distribution in the ring of each node relative to each other. fn _ring_distribution<'a>( - nodes: impl Iterator + 'a, + nodes: impl Iterator + 'a, ) -> impl Iterator + 'a { // TODO: groupby certain intervals // e.g. grouping func: (it * 200.0).roundToInt().toDouble() / 200.0 diff --git a/crates/freenet2-node/src/node/in_memory.rs b/crates/freenet2-node/src/node/in_memory.rs index 4dc083155..1f180d050 100644 --- a/crates/freenet2-node/src/node/in_memory.rs +++ b/crates/freenet2-node/src/node/in_memory.rs @@ -7,7 +7,7 @@ use crate::{ use super::op_state::OpStateStorage; -pub(crate) struct InMemory { +pub(crate) struct NodeInMemory { peer: PeerKey, listening: bool, gateways: Vec, @@ -15,7 +15,7 @@ pub(crate) struct InMemory { pub op_storage: OpStateStorage, } -impl InMemory { +impl NodeInMemory { pub fn build(config: NodeConfig) -> Result { if (config.local_ip.is_none() || config.local_port.is_none()) && config.remote_nodes.is_empty() @@ -24,7 +24,7 @@ impl InMemory { } let peer = PeerKey::from(config.local_key.public()); let conn_manager = MemoryConnManager::new(true, peer, None); - Ok(InMemory { + Ok(NodeInMemory { peer, listening: true, conn_manager, @@ -70,14 +70,16 @@ impl InMemory { match self.conn_manager.recv().await { Ok(msg) => match msg { Message::JoinRing(join_op) => { - join_ring::join_ring_op(&mut self.op_storage, &mut self.conn_manager, join_op) - .await - .unwrap(); + join_ring::join_ring_op( + &mut self.op_storage, + &mut self.conn_manager, + join_op, + ) + .await + .unwrap(); } + Message::Put(_) => todo!(), Message::Canceled(_) => todo!(), - // old: - Message::ProbeRequest(_, _) => todo!(), - Message::ProbeResponse(_, _) => todo!(), }, Err(_) => break Err(()), } diff --git a/crates/freenet2-node/src/node/op_state.rs b/crates/freenet2-node/src/node/op_state.rs index 49ba2bf91..3cd04c793 100644 --- a/crates/freenet2-node/src/node/op_state.rs +++ b/crates/freenet2-node/src/node/op_state.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use crate::{ message::{Transaction, TransactionTypeId}, - operations::{join_ring, OpsMap}, + operations::{join_ring, put, OpsMap}, ring::Ring, }; @@ -37,6 +37,21 @@ impl OpStateStorage { pub fn pop_join_ring_op(&mut self, id: &Transaction) -> Option { self.ops.join_ring.remove(id) } + + pub fn push_put_op(&mut self, id: Transaction, tx: put::PutOp) -> Result<(), OpExecutionError> { + if !matches!(id.tx_type(), TransactionTypeId::Put) { + return Err(OpExecutionError::IncorrectTxType( + TransactionTypeId::Put, + id.tx_type(), + )); + } + self.ops.put.insert(id, tx); + Ok(()) + } + + pub fn pop_put_op(&mut self, id: &Transaction) -> Option { + self.ops.put.remove(id) + } } #[derive(Debug, thiserror::Error, Clone)] @@ -46,6 +61,3 @@ pub(crate) enum OpExecutionError { #[error("failed while processing transaction {0}")] TxUpdateFailure(Transaction), } - -#[cfg(test)] -mod tests {} diff --git a/crates/freenet2-node/src/operations.rs b/crates/freenet2-node/src/operations.rs index baab77b60..b44aec7f6 100644 --- a/crates/freenet2-node/src/operations.rs +++ b/crates/freenet2-node/src/operations.rs @@ -6,9 +6,13 @@ use crate::{ node::OpExecutionError, }; -use self::join_ring::JoinRingOp; +use self::{join_ring::JoinRingOp, put::PutOp}; +pub(crate) mod get; pub(crate) mod join_ring; +pub(crate) mod put; +pub(crate) mod routing; +pub(crate) mod subscribe; pub(crate) struct OperationResult { /// Inhabited if there is a message to return to the other peer. @@ -53,6 +57,7 @@ macro_rules! op_type_enumeration { } op_type_enumeration!(decl struct { - join_ring: JoinRingOp + join_ring: JoinRingOp, + put: PutOp // probe_peers: ProbeOp }); diff --git a/crates/freenet2-node/src/operations/get.rs b/crates/freenet2-node/src/operations/get.rs new file mode 100644 index 000000000..3b0b40a53 --- /dev/null +++ b/crates/freenet2-node/src/operations/get.rs @@ -0,0 +1,10 @@ +use std::marker::PhantomData; + +/// This is just a placeholder for now! +pub(crate) struct GetOp(PhantomData<()>); + +impl GetOp { + pub fn new() -> Self { + GetOp(PhantomData) + } +} diff --git a/crates/freenet2-node/src/operations/join_ring.rs b/crates/freenet2-node/src/operations/join_ring.rs index f64ca6f51..9241e9f09 100644 --- a/crates/freenet2-node/src/operations/join_ring.rs +++ b/crates/freenet2-node/src/operations/join_ring.rs @@ -421,7 +421,7 @@ where ); conn_manager.add_connection(gateway, true); - let tx = Transaction::new(::msg_type_id()); + let tx = Transaction::new(::tx_type_id()); let join_req = Message::from(messages::JoinRingMsg::Req { id: tx, msg: messages::JoinRequest::Initial { @@ -657,12 +657,12 @@ mod tests { match tokio::time::timeout(Duration::from_secs(1), sim_net.recv_net_events()).await { Ok(Some(Ok(event))) => match event.event { EventType::JoinSuccess { gateway, new_node } => { - log::info!("Successful join op between {} and {}", gateway, new_node) + log::info!("Successful join op between {} and {}", gateway, new_node); + Ok(()) } }, - _ => return Err("no event received".into()), + _ => Err("no event received".into()), } - Ok(()) } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/crates/freenet2-node/src/operations/put.rs b/crates/freenet2-node/src/operations/put.rs new file mode 100644 index 000000000..b4ba18aa7 --- /dev/null +++ b/crates/freenet2-node/src/operations/put.rs @@ -0,0 +1,35 @@ +use std::marker::PhantomData; + +use crate::message::Transaction; + +pub(crate) use self::messages::PutMsg; + +/// This is just a placeholder for now! +pub(crate) struct PutOp(PhantomData<()>); + +impl PutOp { + pub fn new() -> Self { + PutOp(PhantomData) + } +} + +mod messages { + use crate::conn_manager::PeerKeyLocation; + + use super::*; + + use serde::{Deserialize, Serialize}; + + #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] + pub(crate) enum PutMsg {} + + impl PutMsg { + pub fn id(&self) -> &Transaction { + todo!() + } + + pub fn sender(&self) -> Option<&PeerKeyLocation> { + todo!() + } + } +} diff --git a/crates/freenet2-node/src/operations/routing.rs b/crates/freenet2-node/src/operations/routing.rs new file mode 100644 index 000000000..af9061b75 --- /dev/null +++ b/crates/freenet2-node/src/operations/routing.rs @@ -0,0 +1,2 @@ +//! The routing mechanism consist on a greedy routing algorithm which just targets +//! the closest peer to the target destination. diff --git a/crates/freenet2-node/src/operations/subscribe.rs b/crates/freenet2-node/src/operations/subscribe.rs new file mode 100644 index 000000000..e69de29bb From ab6d60bf7860d6de32519ad37077d2905449b34d Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Sat, 25 Sep 2021 11:13:18 +0200 Subject: [PATCH 09/14] Make op state pop/push ops general --- crates/freenet2-node/src/node/in_memory.rs | 23 ++++--- crates/freenet2-node/src/node/op_state.rs | 67 ++++++++++--------- crates/freenet2-node/src/operations.rs | 43 ++---------- .../freenet2-node/src/operations/join_ring.rs | 27 +++++--- crates/freenet2-node/src/operations/put.rs | 15 ++++- 5 files changed, 85 insertions(+), 90 deletions(-) diff --git a/crates/freenet2-node/src/node/in_memory.rs b/crates/freenet2-node/src/node/in_memory.rs index 1f180d050..a8e6795d1 100644 --- a/crates/freenet2-node/src/node/in_memory.rs +++ b/crates/freenet2-node/src/node/in_memory.rs @@ -1,7 +1,10 @@ use crate::{ conn_manager::{in_memory::MemoryConnManager, ConnectionBridge, PeerKeyLocation}, message::Message, - operations::join_ring::{self, JoinRingOp}, + operations::{ + join_ring::{self, JoinRingOp}, + put, + }, NodeConfig, PeerKey, }; @@ -69,16 +72,16 @@ impl NodeInMemory { loop { match self.conn_manager.recv().await { Ok(msg) => match msg { - Message::JoinRing(join_op) => { - join_ring::join_ring_op( - &mut self.op_storage, - &mut self.conn_manager, - join_op, - ) - .await - .unwrap(); + Message::JoinRing(op) => { + join_ring::join_ring_op(&mut self.op_storage, &mut self.conn_manager, op) + .await + .unwrap(); + } + Message::Put(op) => { + put::put_op(&mut self.op_storage, &mut self.conn_manager, op) + .await + .unwrap(); } - Message::Put(_) => todo!(), Message::Canceled(_) => todo!(), }, Err(_) => break Err(()), diff --git a/crates/freenet2-node/src/node/op_state.rs b/crates/freenet2-node/src/node/op_state.rs index 3cd04c793..87e294e80 100644 --- a/crates/freenet2-node/src/node/op_state.rs +++ b/crates/freenet2-node/src/node/op_state.rs @@ -1,56 +1,61 @@ -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; use crate::{ message::{Transaction, TransactionTypeId}, - operations::{join_ring, put, OpsMap}, + operations::{ + join_ring::{self, JoinRingOp}, + put::{self, PutOp}, + Operation, + }, ring::Ring, }; pub(crate) struct OpStateStorage { - ops: OpsMap, + join_ring: HashMap, + put: HashMap, pub ring: Arc, } +macro_rules! check_id_op { + ($get_ty:expr, $var:path) => { + if !matches!($get_ty, $var) { + return Err(OpExecutionError::IncorrectTxType( + TransactionTypeId::JoinRing, + $get_ty, + )); + } + }; +} + impl OpStateStorage { pub fn new() -> Self { Self { - ops: OpsMap::new(), + join_ring: HashMap::default(), + put: HashMap::default(), ring: Arc::new(Ring::new()), } } - pub fn push_join_ring_op( - &mut self, - id: Transaction, - tx: join_ring::JoinRingOp, - ) -> Result<(), OpExecutionError> { - if !matches!(id.tx_type(), TransactionTypeId::JoinRing) { - return Err(OpExecutionError::IncorrectTxType( - TransactionTypeId::JoinRing, - id.tx_type(), - )); + pub fn push(&mut self, id: Transaction, op: Operation) -> Result<(), OpExecutionError> { + match op { + Operation::JoinRing(tx) => { + check_id_op!(id.tx_type(), TransactionTypeId::JoinRing); + self.join_ring.insert(id, tx); + } + Operation::Put(tx) => { + check_id_op!(id.tx_type(), TransactionTypeId::Put); + self.put.insert(id, tx); + } } - self.ops.join_ring.insert(id, tx); Ok(()) } - pub fn pop_join_ring_op(&mut self, id: &Transaction) -> Option { - self.ops.join_ring.remove(id) - } - - pub fn push_put_op(&mut self, id: Transaction, tx: put::PutOp) -> Result<(), OpExecutionError> { - if !matches!(id.tx_type(), TransactionTypeId::Put) { - return Err(OpExecutionError::IncorrectTxType( - TransactionTypeId::Put, - id.tx_type(), - )); + pub fn pop(&mut self, id: &Transaction) -> Option { + match id.tx_type() { + TransactionTypeId::JoinRing => self.join_ring.remove(id).map(Operation::JoinRing), + TransactionTypeId::Put => self.put.remove(id).map(Operation::Put), + TransactionTypeId::Canceled => todo!(), } - self.ops.put.insert(id, tx); - Ok(()) - } - - pub fn pop_put_op(&mut self, id: &Transaction) -> Option { - self.ops.put.remove(id) } } diff --git a/crates/freenet2-node/src/operations.rs b/crates/freenet2-node/src/operations.rs index b44aec7f6..66da76167 100644 --- a/crates/freenet2-node/src/operations.rs +++ b/crates/freenet2-node/src/operations.rs @@ -1,12 +1,4 @@ -use serde::{Deserialize, Serialize}; - -use crate::{ - conn_manager, - message::{Message, Transaction}, - node::OpExecutionError, -}; - -use self::{join_ring::JoinRingOp, put::PutOp}; +use crate::{conn_manager, message::Message, node::OpExecutionError}; pub(crate) mod get; pub(crate) mod join_ring; @@ -21,6 +13,11 @@ pub(crate) struct OperationResult { pub state: Option, } +pub(crate) enum Operation { + JoinRing(join_ring::JoinRingOp), + Put(put::PutOp), +} + #[derive(Debug, Default)] pub struct ProbeOp; @@ -33,31 +30,3 @@ pub(crate) enum OpError { #[error("illegal awaiting state")] IllegalStateTransition, } - -macro_rules! op_type_enumeration { - (decl struct { $($field:ident: $var:tt),+ } ) => { - #[repr(u8)] - #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy, Serialize, Deserialize)] - pub(crate) enum OperationType { - $($var,)+ - } - - pub(crate) struct OpsMap { - $( pub $field: std::collections::HashMap),+, - } - - impl OpsMap { - pub fn new() -> Self { - Self { - $( $field: std::collections::HashMap::new()),+, - } - } - } - }; -} - -op_type_enumeration!(decl struct { - join_ring: JoinRingOp, - put: PutOp - // probe_peers: ProbeOp -}); diff --git a/crates/freenet2-node/src/operations/join_ring.rs b/crates/freenet2-node/src/operations/join_ring.rs index 9241e9f09..db40654bd 100644 --- a/crates/freenet2-node/src/operations/join_ring.rs +++ b/crates/freenet2-node/src/operations/join_ring.rs @@ -7,6 +7,7 @@ use crate::{ conn_manager::{self, ConnectionBridge, PeerKeyLocation}, message::{Message, Transaction, TransactionType}, node::{OpExecutionError, OpStateStorage}, + operations::Operation, ring::{Location, Ring}, PeerKey, }; @@ -145,15 +146,19 @@ where { let sender; let tx = *join_op.id(); - let result = if let Some(state) = op_storage.pop_join_ring_op(join_op.id()) { - sender = join_op.sender().cloned(); - // was an existing operation, the other peer messaged back - update_state(conn_manager, state, join_op, &op_storage.ring).await - } else { - sender = join_op.sender().cloned(); - // new request to join from this node, initialize the machine - let machine = JoinRingOp(StateMachine::new()); - update_state(conn_manager, machine, join_op, &op_storage.ring).await + let result = match op_storage.pop(join_op.id()) { + Some(Operation::JoinRing(state)) => { + sender = join_op.sender().cloned(); + // was an existing operation, the other peer messaged back + update_state(conn_manager, state, join_op, &op_storage.ring).await + } + Some(_) => return Err(OpExecutionError::TxUpdateFailure(tx).into()), + None => { + sender = join_op.sender().cloned(); + // new request to join from this node, initialize the machine + let machine = JoinRingOp(StateMachine::new()); + update_state(conn_manager, machine, join_op, &op_storage.ring).await + } }; match result { @@ -173,7 +178,7 @@ where if let Some(target) = msg.sender().cloned() { conn_manager.send(&target, msg).await?; } - op_storage.push_join_ring_op(id, updated_state)?; + op_storage.push(id, Operation::JoinRing(updated_state))?; } Ok(OperationResult { return_msg: Some(msg), @@ -441,7 +446,7 @@ where // .0 // .consume(&JoinRingMsg::Req { , this_peer }) // .map_err(|_| OpError::IllegalStateTransition)?; - op_storage.push_join_ring_op(tx, join_op)?; + op_storage.push(tx, Operation::JoinRing(join_op))?; Ok(()) } diff --git a/crates/freenet2-node/src/operations/put.rs b/crates/freenet2-node/src/operations/put.rs index b4ba18aa7..7918da91a 100644 --- a/crates/freenet2-node/src/operations/put.rs +++ b/crates/freenet2-node/src/operations/put.rs @@ -1,9 +1,11 @@ use std::marker::PhantomData; -use crate::message::Transaction; +use crate::{conn_manager::ConnectionBridge, message::Transaction, node::OpStateStorage}; pub(crate) use self::messages::PutMsg; +use super::OpError; + /// This is just a placeholder for now! pub(crate) struct PutOp(PhantomData<()>); @@ -13,6 +15,17 @@ impl PutOp { } } +pub(crate) async fn put_op( + op_storage: &mut OpStateStorage, + conn_manager: &mut CB, + join_op: PutMsg, +) -> Result<(), OpError> +where + CB: ConnectionBridge, +{ + Ok(()) +} + mod messages { use crate::conn_manager::PeerKeyLocation; From e34d3cf826c0b39dab054f6d2e1e81e0e4da9487 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Sat, 25 Sep 2021 12:59:26 +0200 Subject: [PATCH 10/14] Receive user inputs concurrently --- crates/freenet2-node/src/conn_manager.rs | 2 + crates/freenet2-node/src/lib.rs | 1 + crates/freenet2-node/src/node.rs | 5 +- crates/freenet2-node/src/node/in_memory.rs | 54 ++++++++++++------- crates/freenet2-node/src/node/op_state.rs | 6 +-- .../freenet2-node/src/operations/join_ring.rs | 4 +- crates/freenet2-node/src/operations/put.rs | 18 ++++++- crates/freenet2-node/src/user_events.rs | 29 ++++++++++ 8 files changed, 87 insertions(+), 32 deletions(-) create mode 100644 crates/freenet2-node/src/user_events.rs diff --git a/crates/freenet2-node/src/conn_manager.rs b/crates/freenet2-node/src/conn_manager.rs index d9e7bb2ed..81497ff3f 100644 --- a/crates/freenet2-node/src/conn_manager.rs +++ b/crates/freenet2-node/src/conn_manager.rs @@ -23,6 +23,8 @@ pub(crate) type Result = StdResult; pub(crate) trait ConnectionBridge { fn add_connection(&mut self, peer: PeerKeyLocation, unsolicited: bool); + /// # Cancellation Safety + /// This async fn must be cancellation safe! async fn recv(&self) -> Result; async fn send(&self, target: &PeerKeyLocation, msg: Message) -> Result<()>; diff --git a/crates/freenet2-node/src/lib.rs b/crates/freenet2-node/src/lib.rs index 58c040865..32885ed08 100644 --- a/crates/freenet2-node/src/lib.rs +++ b/crates/freenet2-node/src/lib.rs @@ -5,6 +5,7 @@ mod node; mod operations; // mod probe_proto; mod ring; +mod user_events; pub use conn_manager::PeerKey; pub use node::NodeConfig; diff --git a/crates/freenet2-node/src/node.rs b/crates/freenet2-node/src/node.rs index 446dfd069..8221ad07e 100644 --- a/crates/freenet2-node/src/node.rs +++ b/crates/freenet2-node/src/node.rs @@ -186,10 +186,7 @@ fn multiaddr_from_connection(conn: (IpAddr, u16)) -> Multiaddr { #[cfg(test)] pub mod test_utils { - use std::{ - collections::HashMap, - net::{Ipv4Addr, Ipv6Addr, SocketAddr, TcpListener}, - }; + use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, TcpListener}; use libp2p::identity; use rand::Rng; diff --git a/crates/freenet2-node/src/node/in_memory.rs b/crates/freenet2-node/src/node/in_memory.rs index a8e6795d1..08a1bf76d 100644 --- a/crates/freenet2-node/src/node/in_memory.rs +++ b/crates/freenet2-node/src/node/in_memory.rs @@ -1,21 +1,23 @@ use crate::{ conn_manager::{in_memory::MemoryConnManager, ConnectionBridge, PeerKeyLocation}, message::Message, - operations::{ - join_ring::{self, JoinRingOp}, - put, - }, + operations::{join_ring, put}, + user_events::{test_utils::MemoryEventsGen, UserEvent, UserEventsProxy}, NodeConfig, PeerKey, }; use super::op_state::OpStateStorage; -pub(crate) struct NodeInMemory { +pub(crate) struct NodeInMemory +where + Ev: UserEventsProxy, +{ peer: PeerKey, listening: bool, gateways: Vec, pub conn_manager: MemoryConnManager, pub op_storage: OpStateStorage, + user_events: Ev, } impl NodeInMemory { @@ -33,6 +35,7 @@ impl NodeInMemory { conn_manager, op_storage: OpStateStorage::new(), gateways: vec![], + user_events: MemoryEventsGen {}, }) } @@ -47,7 +50,7 @@ impl NodeInMemory { // the idea here is to limit the amount of gateways being contacted that's why shuffling is required for gateway in &self.gateways { // initiate join action action per each gateway - let op = JoinRingOp::new( + let op = join_ring::JoinRingOp::new( PeerKeyLocation { peer: self.peer, location: None, @@ -70,21 +73,34 @@ impl NodeInMemory { } loop { - match self.conn_manager.recv().await { - Ok(msg) => match msg { - Message::JoinRing(op) => { - join_ring::join_ring_op(&mut self.op_storage, &mut self.conn_manager, op) - .await - .unwrap(); + tokio::select! { + msg = self.conn_manager.recv() => { + match msg { + Ok(msg) => match msg { + Message::JoinRing(op) => { + join_ring::join_ring_op(&mut self.op_storage, &mut self.conn_manager, op) + .await + .unwrap(); + } + Message::Put(op) => { + put::put_op(&mut self.op_storage, &mut self.conn_manager, op) + .await + .unwrap(); + } + Message::Canceled(_) => todo!(), + }, + Err(_) => break Err(()), } - Message::Put(op) => { - put::put_op(&mut self.op_storage, &mut self.conn_manager, op) - .await - .unwrap(); + } + usr_event = self.user_events.recv() => { + match usr_event { + UserEvent::Put {key, value }=> { + // Initialize a put op. + let op = put::PutOp::new(); + put::request_put(&mut self.op_storage, &mut self.conn_manager, op).await.unwrap(); + } } - Message::Canceled(_) => todo!(), - }, - Err(_) => break Err(()), + } } } } diff --git a/crates/freenet2-node/src/node/op_state.rs b/crates/freenet2-node/src/node/op_state.rs index 87e294e80..e63dcf932 100644 --- a/crates/freenet2-node/src/node/op_state.rs +++ b/crates/freenet2-node/src/node/op_state.rs @@ -2,11 +2,7 @@ use std::{collections::HashMap, sync::Arc}; use crate::{ message::{Transaction, TransactionTypeId}, - operations::{ - join_ring::{self, JoinRingOp}, - put::{self, PutOp}, - Operation, - }, + operations::{join_ring::JoinRingOp, put::PutOp, Operation}, ring::Ring, }; diff --git a/crates/freenet2-node/src/operations/join_ring.rs b/crates/freenet2-node/src/operations/join_ring.rs index db40654bd..0a17c0d4e 100644 --- a/crates/freenet2-node/src/operations/join_ring.rs +++ b/crates/freenet2-node/src/operations/join_ring.rs @@ -134,8 +134,8 @@ impl JRState { /// Join ring routine, called upon processing a request to join or performing /// a join operation for this node. /// -/// # Arguments -/// - join_op: no nodes +/// # Cancellation Safety +/// This future is not cancellation safe. pub(crate) async fn join_ring_op( op_storage: &mut OpStateStorage, conn_manager: &mut CB, diff --git a/crates/freenet2-node/src/operations/put.rs b/crates/freenet2-node/src/operations/put.rs index 7918da91a..999cf4ea7 100644 --- a/crates/freenet2-node/src/operations/put.rs +++ b/crates/freenet2-node/src/operations/put.rs @@ -18,7 +18,7 @@ impl PutOp { pub(crate) async fn put_op( op_storage: &mut OpStateStorage, conn_manager: &mut CB, - join_op: PutMsg, + put_op: PutMsg, ) -> Result<(), OpError> where CB: ConnectionBridge, @@ -26,6 +26,18 @@ where Ok(()) } +/// Request to insert/update a value into a contract. +pub(crate) async fn request_put( + op_storage: &mut OpStateStorage, + conn_manager: &mut CB, + put_op: PutOp, +) -> Result<(), OpError> +where + CB: ConnectionBridge, +{ + todo!() +} + mod messages { use crate::conn_manager::PeerKeyLocation; @@ -34,7 +46,9 @@ mod messages { use serde::{Deserialize, Serialize}; #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] - pub(crate) enum PutMsg {} + pub(crate) enum PutMsg { + RouteValue { key: Vec, value: Vec }, + } impl PutMsg { pub fn id(&self) -> &Transaction { diff --git a/crates/freenet2-node/src/user_events.rs b/crates/freenet2-node/src/user_events.rs new file mode 100644 index 000000000..10f478ef6 --- /dev/null +++ b/crates/freenet2-node/src/user_events.rs @@ -0,0 +1,29 @@ +#[async_trait::async_trait] +pub(crate) trait UserEventsProxy { + async fn recv(&self) -> UserEvent; +} + +pub(crate) enum UserEvent { + /// Update or insert a new value in a contract corresponding with the provided key + Put { + /// Hash key of the contract. + key: Vec, + /// Value to upsert in the contract. + value: Vec, + }, +} + +pub(crate) mod test_utils { + use super::*; + + pub(crate) struct MemoryEventsGen {} + + #[async_trait::async_trait] + impl UserEventsProxy for MemoryEventsGen { + /// # Cancellation Safety + /// This future must be safe to cancel. + async fn recv(&self) -> UserEvent { + todo!() + } + } +} From e7f62f9a68eddc94afd97b808d23ce952af0a6e5 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Sat, 25 Sep 2021 18:57:29 +0200 Subject: [PATCH 11/14] Add routing func in ring type --- crates/freenet2-node/src/conn_manager.rs | 9 +-- crates/freenet2-node/src/node/in_memory.rs | 2 +- crates/freenet2-node/src/operations.rs | 1 - .../freenet2-node/src/operations/join_ring.rs | 10 +-- crates/freenet2-node/src/operations/put.rs | 51 +++++++++++-- .../freenet2-node/src/operations/routing.rs | 2 - crates/freenet2-node/src/ring.rs | 72 ++++++++++++++++--- 7 files changed, 118 insertions(+), 29 deletions(-) delete mode 100644 crates/freenet2-node/src/operations/routing.rs diff --git a/crates/freenet2-node/src/conn_manager.rs b/crates/freenet2-node/src/conn_manager.rs index 81497ff3f..415839320 100644 --- a/crates/freenet2-node/src/conn_manager.rs +++ b/crates/freenet2-node/src/conn_manager.rs @@ -36,15 +36,8 @@ pub(crate) trait Transport { fn location(&self) -> Option; } -#[derive(Debug, PartialEq, Eq, Hash)] -struct Peer { - addr: SocketAddr, - port: u16, - label: Option, -} - #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] -pub struct PeerKey(PeerId); +pub struct PeerKey(pub(crate) PeerId); impl Display for PeerKey { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { diff --git a/crates/freenet2-node/src/node/in_memory.rs b/crates/freenet2-node/src/node/in_memory.rs index 08a1bf76d..2a62c4cdd 100644 --- a/crates/freenet2-node/src/node/in_memory.rs +++ b/crates/freenet2-node/src/node/in_memory.rs @@ -96,7 +96,7 @@ impl NodeInMemory { match usr_event { UserEvent::Put {key, value }=> { // Initialize a put op. - let op = put::PutOp::new(); + let op = put::PutOp::new(key, value); put::request_put(&mut self.op_storage, &mut self.conn_manager, op).await.unwrap(); } } diff --git a/crates/freenet2-node/src/operations.rs b/crates/freenet2-node/src/operations.rs index 66da76167..efa14f37b 100644 --- a/crates/freenet2-node/src/operations.rs +++ b/crates/freenet2-node/src/operations.rs @@ -3,7 +3,6 @@ use crate::{conn_manager, message::Message, node::OpExecutionError}; pub(crate) mod get; pub(crate) mod join_ring; pub(crate) mod put; -pub(crate) mod routing; pub(crate) mod subscribe; pub(crate) struct OperationResult { diff --git a/crates/freenet2-node/src/operations/join_ring.rs b/crates/freenet2-node/src/operations/join_ring.rs index 0a17c0d4e..981acbca5 100644 --- a/crates/freenet2-node/src/operations/join_ring.rs +++ b/crates/freenet2-node/src/operations/join_ring.rs @@ -14,7 +14,7 @@ use crate::{ pub(crate) use self::messages::{JoinRequest, JoinResponse, JoinRingMsg}; -pub(crate) struct JoinRingOp(StateMachine); +pub(crate) struct JoinRingOp(StateMachine); impl JoinRingOp { pub fn new(this_peer: PeerKeyLocation, gateway: PeerKeyLocation) -> Self { @@ -23,9 +23,9 @@ impl JoinRingOp { } #[derive(Debug)] -struct InternalJROp; +struct JROpSM; -impl StateMachineImpl for InternalJROp { +impl StateMachineImpl for JROpSM { type Input = JoinRingMsg; type State = JRState; @@ -610,7 +610,7 @@ mod tests { location: None, }; - let mut join_op_host_1 = StateMachine::::new(); + let mut join_op_host_1 = StateMachine::::new(); let res = join_op_host_1 .consume(&JoinRingMsg::Req { id, @@ -631,7 +631,7 @@ mod tests { assert_eq!(res, expected); assert!(matches!(join_op_host_1.state(), JRState::Connecting(_))); - let mut join_op_host_2 = StateMachine::::new(); + let mut join_op_host_2 = StateMachine::::new(); let res = join_op_host_2.consume(&res).unwrap().unwrap(); let expected = JoinRingMsg::Connected; assert_eq!(res, expected); diff --git a/crates/freenet2-node/src/operations/put.rs b/crates/freenet2-node/src/operations/put.rs index 999cf4ea7..4adf29bef 100644 --- a/crates/freenet2-node/src/operations/put.rs +++ b/crates/freenet2-node/src/operations/put.rs @@ -1,4 +1,4 @@ -use std::marker::PhantomData; +use rust_fsm::{StateMachine, StateMachineImpl}; use crate::{conn_manager::ConnectionBridge, message::Transaction, node::OpStateStorage}; @@ -7,14 +7,54 @@ pub(crate) use self::messages::PutMsg; use super::OpError; /// This is just a placeholder for now! -pub(crate) struct PutOp(PhantomData<()>); +pub(crate) struct PutOp(StateMachine); impl PutOp { - pub fn new() -> Self { - PutOp(PhantomData) + pub fn new(key: Vec, value: Vec) -> Self { + let mut state = StateMachine::new(); + state.consume(&PutMsg::RouteValue { key, value }).unwrap(); + PutOp(state) } } +struct PutOpSM; + +impl StateMachineImpl for PutOpSM { + type Input = PutMsg; + + type State = PutState; + + type Output = PutMsg; + + const INITIAL_STATE: Self::State = PutState::Initializing; + + fn transition(state: &Self::State, input: &Self::Input) -> Option { + match (state, input) { + (PutState::Initializing, PutMsg::RouteValue { key, .. }) => { + Some(PutState::Requesting { key: key.clone() }) + } + _ => None, + } + } + + fn output(state: &Self::State, input: &Self::Input) -> Option { + match (state, input) { + (PutState::Initializing, PutMsg::RouteValue { key, value }) => { + Some(PutMsg::RouteValue { + key: key.clone(), + value: value.clone(), + }) + } + _ => None, + } + } +} + +enum PutState { + Initializing, + Requesting { key: Vec }, +} + pub(crate) async fn put_op( op_storage: &mut OpStateStorage, conn_manager: &mut CB, @@ -35,6 +75,9 @@ pub(crate) async fn request_put( where CB: ConnectionBridge, { + // the initial request must provide: + // - a location in the network where the contract resides + // - and the values to put todo!() } diff --git a/crates/freenet2-node/src/operations/routing.rs b/crates/freenet2-node/src/operations/routing.rs deleted file mode 100644 index af9061b75..000000000 --- a/crates/freenet2-node/src/operations/routing.rs +++ /dev/null @@ -1,2 +0,0 @@ -//! The routing mechanism consist on a greedy routing algorithm which just targets -//! the closest peer to the target destination. diff --git a/crates/freenet2-node/src/ring.rs b/crates/freenet2-node/src/ring.rs index 76b4bcfe5..56a02143d 100644 --- a/crates/freenet2-node/src/ring.rs +++ b/crates/freenet2-node/src/ring.rs @@ -1,4 +1,14 @@ //! Ring protocol logic and supporting types. +//! +//! # Routing +//! The routing mechanism consist in a greedy routing algorithm which just targets +//! the closest location to the target destination iteratively in each hop, until it reaches +//! the destination. +//! +//! Path is limited to local knowledge, at any given point only 3 data points are known: +//! - previous node +//! - next node +//! - final location use std::{collections::BTreeMap, convert::TryFrom, fmt::Display, hash::Hasher}; @@ -56,18 +66,24 @@ impl Ring { } pub fn median_distance_to(&self, location: &Location) -> Distance { - let mut conn_by_dist = self.connections_by_distance(location); - conn_by_dist.sort_by_key(|(k, _)| *k); + let connections = self.connections_by_location.read(); + let mut conn_by_dist: Vec<_> = connections + .keys() + .map(|key| key.distance(location)) + .collect(); + conn_by_dist.sort_unstable(); let idx = self.connections_by_location.read().len() / 2; - conn_by_dist[idx].0 + conn_by_dist[idx] } - pub fn connections_by_distance(&self, to: &Location) -> Vec<(Distance, PeerKeyLocation)> { - self.connections_by_location - .read() + pub fn routing(&self, target: &Location) -> Option<(Location, PeerKeyLocation)> { + let connections = self.connections_by_location.read(); + let mut conn_by_dist: Vec<_> = connections .iter() - .map(|(key, peer)| (key.distance(to), *peer)) - .collect() + .map(|(loc, peer)| (loc.distance(target), (loc, peer))) + .collect(); + conn_by_dist.sort_by_key(|&(dist, _)| dist); + conn_by_dist.first().map(|(_, v)| (*v.0, *v.1)) } pub fn random_peer(&self, filter_fn: F) -> Option @@ -165,3 +181,43 @@ pub(crate) enum RingProtoError { #[error(transparent)] ConnError(#[from] Box), } + +#[cfg(test)] +mod tests { + use libp2p::PeerId; + + use crate::PeerKey; + + use super::*; + + #[test] + fn location_dist() { + let l0 = Location(0.); + let l1 = Location(0.25); + assert!(l0.distance(&l1) == Location(0.25)); + + let l0 = Location(0.75); + let l1 = Location(0.50); + assert!(l0.distance(&l1) == Location(0.25)); + } + + #[test] + fn find_closest() { + let ring = Ring::new(); + { + let conns = &mut *ring.connections_by_location.write(); + let def_peer = PeerKeyLocation { + peer: PeerKey(PeerId::random()), + location: None, + }; + conns.insert(Location(0.3), def_peer); + conns.insert(Location(0.5), def_peer); + conns.insert(Location(0.0), def_peer); + } + + assert_eq!(Location(0.0), ring.routing(&Location(0.9)).unwrap().0); + assert_eq!(Location(0.0), ring.routing(&Location(0.1)).unwrap().0); + assert_eq!(Location(0.5), ring.routing(&Location(0.41)).unwrap().0); + assert_eq!(Location(0.3), ring.routing(&Location(0.39)).unwrap().0); + } +} From a05ea18ef347c0316c4ee62ccca6ab359902d573 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Sat, 25 Sep 2021 19:57:42 +0200 Subject: [PATCH 12/14] Add get value op --- crates/freenet2-node/src/message.rs | 9 +- crates/freenet2-node/src/node/in_memory.rs | 14 ++- crates/freenet2-node/src/node/op_state.rs | 9 +- crates/freenet2-node/src/operations.rs | 1 + crates/freenet2-node/src/operations/get.rs | 100 ++++++++++++++++++++- crates/freenet2-node/src/operations/put.rs | 2 +- crates/freenet2-node/src/user_events.rs | 7 +- 7 files changed, 131 insertions(+), 11 deletions(-) diff --git a/crates/freenet2-node/src/message.rs b/crates/freenet2-node/src/message.rs index d12307092..fecabc1ef 100644 --- a/crates/freenet2-node/src/message.rs +++ b/crates/freenet2-node/src/message.rs @@ -5,7 +5,7 @@ use uuid::Uuid; use crate::{ conn_manager::PeerKeyLocation, - operations::{join_ring::JoinRingMsg, put::PutMsg}, + operations::{get::GetMsg, join_ring::JoinRingMsg, put::PutMsg}, ring::Location, }; pub(crate) use sealed_msg_type::TransactionTypeId; @@ -76,6 +76,7 @@ mod sealed_msg_type { pub(crate) enum TransactionTypeId { JoinRing, Put, + Get, Canceled, } @@ -100,6 +101,7 @@ mod sealed_msg_type { transaction_type_enumeration!(decl struct { JoinRing -> JoinRingMsg, Put -> PutMsg, + Get -> GetMsg, Canceled -> Transaction }); } @@ -108,6 +110,7 @@ mod sealed_msg_type { pub(crate) enum Message { JoinRing(JoinRingMsg), Put(PutMsg), + Get(GetMsg), /// Failed a transaction, informing of cancellation. Canceled(Transaction), } @@ -122,6 +125,7 @@ impl Message { match self { JoinRing(op) => op.id(), Put(op) => op.id(), + Get(op) => op.id(), Canceled(tx) => tx, } } @@ -130,8 +134,9 @@ impl Message { use Message::*; match self { JoinRing(op) => op.sender(), - Canceled(_) => None, Put(op) => op.sender(), + Get(op) => op.sender(), + Canceled(_) => None, } } } diff --git a/crates/freenet2-node/src/node/in_memory.rs b/crates/freenet2-node/src/node/in_memory.rs index 2a62c4cdd..392c2113b 100644 --- a/crates/freenet2-node/src/node/in_memory.rs +++ b/crates/freenet2-node/src/node/in_memory.rs @@ -1,7 +1,7 @@ use crate::{ conn_manager::{in_memory::MemoryConnManager, ConnectionBridge, PeerKeyLocation}, message::Message, - operations::{join_ring, put}, + operations::{get, join_ring, put}, user_events::{test_utils::MemoryEventsGen, UserEvent, UserEventsProxy}, NodeConfig, PeerKey, }; @@ -87,6 +87,11 @@ impl NodeInMemory { .await .unwrap(); } + Message::Get(op) => { + get::get_op(&mut self.op_storage, &mut self.conn_manager, op) + .await + .unwrap(); + } Message::Canceled(_) => todo!(), }, Err(_) => break Err(()), @@ -94,11 +99,16 @@ impl NodeInMemory { } usr_event = self.user_events.recv() => { match usr_event { - UserEvent::Put {key, value }=> { + UserEvent::Put { key, value } => { // Initialize a put op. let op = put::PutOp::new(key, value); put::request_put(&mut self.op_storage, &mut self.conn_manager, op).await.unwrap(); } + UserEvent::Get { key } => { + // Initialize a get op. + let op = get::GetOp::new(key); + get::request_get(&mut self.op_storage, &mut self.conn_manager, op).await.unwrap(); + } } } } diff --git a/crates/freenet2-node/src/node/op_state.rs b/crates/freenet2-node/src/node/op_state.rs index e63dcf932..1da40c5af 100644 --- a/crates/freenet2-node/src/node/op_state.rs +++ b/crates/freenet2-node/src/node/op_state.rs @@ -2,13 +2,14 @@ use std::{collections::HashMap, sync::Arc}; use crate::{ message::{Transaction, TransactionTypeId}, - operations::{join_ring::JoinRingOp, put::PutOp, Operation}, + operations::{get::GetOp, join_ring::JoinRingOp, put::PutOp, Operation}, ring::Ring, }; pub(crate) struct OpStateStorage { join_ring: HashMap, put: HashMap, + get: HashMap, pub ring: Arc, } @@ -28,6 +29,7 @@ impl OpStateStorage { Self { join_ring: HashMap::default(), put: HashMap::default(), + get: HashMap::default(), ring: Arc::new(Ring::new()), } } @@ -42,6 +44,10 @@ impl OpStateStorage { check_id_op!(id.tx_type(), TransactionTypeId::Put); self.put.insert(id, tx); } + Operation::Get(tx) => { + check_id_op!(id.tx_type(), TransactionTypeId::Put); + self.get.insert(id, tx); + } } Ok(()) } @@ -50,6 +56,7 @@ impl OpStateStorage { match id.tx_type() { TransactionTypeId::JoinRing => self.join_ring.remove(id).map(Operation::JoinRing), TransactionTypeId::Put => self.put.remove(id).map(Operation::Put), + TransactionTypeId::Get => self.get.remove(id).map(Operation::Get), TransactionTypeId::Canceled => todo!(), } } diff --git a/crates/freenet2-node/src/operations.rs b/crates/freenet2-node/src/operations.rs index efa14f37b..3234a4324 100644 --- a/crates/freenet2-node/src/operations.rs +++ b/crates/freenet2-node/src/operations.rs @@ -15,6 +15,7 @@ pub(crate) struct OperationResult { pub(crate) enum Operation { JoinRing(join_ring::JoinRingOp), Put(put::PutOp), + Get(get::GetOp), } #[derive(Debug, Default)] diff --git a/crates/freenet2-node/src/operations/get.rs b/crates/freenet2-node/src/operations/get.rs index 3b0b40a53..d20da92c2 100644 --- a/crates/freenet2-node/src/operations/get.rs +++ b/crates/freenet2-node/src/operations/get.rs @@ -1,10 +1,102 @@ -use std::marker::PhantomData; +use rust_fsm::{StateMachine, StateMachineImpl}; + +use crate::{conn_manager::ConnectionBridge, message::Transaction, node::OpStateStorage}; + +pub(crate) use self::messages::GetMsg; + +use super::OpError; /// This is just a placeholder for now! -pub(crate) struct GetOp(PhantomData<()>); +pub(crate) struct GetOp(StateMachine); impl GetOp { - pub fn new() -> Self { - GetOp(PhantomData) + pub fn new(key: Vec) -> Self { + let mut state = StateMachine::new(); + state.consume(&GetMsg::FetchRouting { key }).unwrap(); + GetOp(state) + } +} + +struct GetOpSM; + +impl StateMachineImpl for GetOpSM { + type Input = GetMsg; + + type State = GetState; + + type Output = GetMsg; + + const INITIAL_STATE: Self::State = GetState::Initializing; + + fn transition(state: &Self::State, inget: &Self::Input) -> Option { + match (state, inget) { + (GetState::Initializing, GetMsg::FetchRouting { key }) => { + Some(GetState::Requesting { key: key.clone() }) + } + _ => None, + } + } + + fn output(state: &Self::State, inget: &Self::Input) -> Option { + match (state, inget) { + (GetState::Initializing, GetMsg::FetchRouting { key }) => { + Some(GetMsg::FetchRouting { key: key.clone() }) + } + _ => None, + } + } +} + +enum GetState { + Initializing, + Requesting { key: Vec }, +} + +pub(crate) async fn get_op( + op_storage: &mut OpStateStorage, + conn_manager: &mut CB, + get_op: GetMsg, +) -> Result<(), OpError> +where + CB: ConnectionBridge, +{ + Ok(()) +} + +/// Request to get the current value from a contract. +pub(crate) async fn request_get( + op_storage: &mut OpStateStorage, + conn_manager: &mut CB, + get_op: GetOp, +) -> Result<(), OpError> +where + CB: ConnectionBridge, +{ + // the initial request must provide: + // - a location in the network where the contract resides + // - and the value to get + todo!() +} + +mod messages { + use crate::conn_manager::PeerKeyLocation; + + use super::*; + + use serde::{Deserialize, Serialize}; + + #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] + pub(crate) enum GetMsg { + FetchRouting { key: Vec }, + } + + impl GetMsg { + pub fn id(&self) -> &Transaction { + todo!() + } + + pub fn sender(&self) -> Option<&PeerKeyLocation> { + todo!() + } } } diff --git a/crates/freenet2-node/src/operations/put.rs b/crates/freenet2-node/src/operations/put.rs index 4adf29bef..da8a469df 100644 --- a/crates/freenet2-node/src/operations/put.rs +++ b/crates/freenet2-node/src/operations/put.rs @@ -77,7 +77,7 @@ where { // the initial request must provide: // - a location in the network where the contract resides - // - and the values to put + // - and the value to put todo!() } diff --git a/crates/freenet2-node/src/user_events.rs b/crates/freenet2-node/src/user_events.rs index 10f478ef6..b17d43fca 100644 --- a/crates/freenet2-node/src/user_events.rs +++ b/crates/freenet2-node/src/user_events.rs @@ -4,13 +4,18 @@ pub(crate) trait UserEventsProxy { } pub(crate) enum UserEvent { - /// Update or insert a new value in a contract corresponding with the provided key + /// Update or insert a new value in a contract corresponding with the provided key. Put { /// Hash key of the contract. key: Vec, /// Value to upsert in the contract. value: Vec, }, + /// Fetch the current value from a contrart corresponding to the provided key. + Get { + /// Hash key of the contract. + key: Vec, + }, } pub(crate) mod test_utils { From 92608bd7b6b2378065346c7eefa264ea2707b460 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Sat, 25 Sep 2021 20:18:58 +0200 Subject: [PATCH 13/14] Rename dir --- .../Cargo.toml | 0 .../src/bin/gateway-deploy.rs | 0 .../src/config.rs | 0 .../src/conn_manager.rs | 0 .../src/conn_manager/in_memory.rs | 0 .../src/lib.rs | 1 - .../src/message.rs | 2 ++ .../src/node.rs | 20 ++++++++++--------- .../src/node/in_memory.rs | 4 ++-- .../src/node/libp2p_impl.rs | 0 .../src/node/op_state.rs | 0 .../src/operations.rs | 0 .../src/operations/get.rs | 0 .../src/operations/join_ring.rs | 11 +++++----- .../src/operations/put.rs | 0 .../src/operations/subscribe.rs | 0 .../src/probe_proto.rs | 0 .../src/ring.rs | 3 +-- .../src/user_events.rs | 0 .../tests/node_startup.rs | 0 20 files changed, 21 insertions(+), 20 deletions(-) rename crates/{freenet2-node => locutus-node}/Cargo.toml (100%) rename crates/{freenet2-node => locutus-node}/src/bin/gateway-deploy.rs (100%) rename crates/{freenet2-node => locutus-node}/src/config.rs (100%) rename crates/{freenet2-node => locutus-node}/src/conn_manager.rs (100%) rename crates/{freenet2-node => locutus-node}/src/conn_manager/in_memory.rs (100%) rename crates/{freenet2-node => locutus-node}/src/lib.rs (86%) rename crates/{freenet2-node => locutus-node}/src/message.rs (98%) rename crates/{freenet2-node => locutus-node}/src/node.rs (94%) rename crates/{freenet2-node => locutus-node}/src/node/in_memory.rs (98%) rename crates/{freenet2-node => locutus-node}/src/node/libp2p_impl.rs (100%) rename crates/{freenet2-node => locutus-node}/src/node/op_state.rs (100%) rename crates/{freenet2-node => locutus-node}/src/operations.rs (100%) rename crates/{freenet2-node => locutus-node}/src/operations/get.rs (100%) rename crates/{freenet2-node => locutus-node}/src/operations/join_ring.rs (98%) rename crates/{freenet2-node => locutus-node}/src/operations/put.rs (100%) rename crates/{freenet2-node => locutus-node}/src/operations/subscribe.rs (100%) rename crates/{freenet2-node => locutus-node}/src/probe_proto.rs (100%) rename crates/{freenet2-node => locutus-node}/src/ring.rs (99%) rename crates/{freenet2-node => locutus-node}/src/user_events.rs (100%) rename crates/{freenet2-node => locutus-node}/tests/node_startup.rs (100%) diff --git a/crates/freenet2-node/Cargo.toml b/crates/locutus-node/Cargo.toml similarity index 100% rename from crates/freenet2-node/Cargo.toml rename to crates/locutus-node/Cargo.toml diff --git a/crates/freenet2-node/src/bin/gateway-deploy.rs b/crates/locutus-node/src/bin/gateway-deploy.rs similarity index 100% rename from crates/freenet2-node/src/bin/gateway-deploy.rs rename to crates/locutus-node/src/bin/gateway-deploy.rs diff --git a/crates/freenet2-node/src/config.rs b/crates/locutus-node/src/config.rs similarity index 100% rename from crates/freenet2-node/src/config.rs rename to crates/locutus-node/src/config.rs diff --git a/crates/freenet2-node/src/conn_manager.rs b/crates/locutus-node/src/conn_manager.rs similarity index 100% rename from crates/freenet2-node/src/conn_manager.rs rename to crates/locutus-node/src/conn_manager.rs diff --git a/crates/freenet2-node/src/conn_manager/in_memory.rs b/crates/locutus-node/src/conn_manager/in_memory.rs similarity index 100% rename from crates/freenet2-node/src/conn_manager/in_memory.rs rename to crates/locutus-node/src/conn_manager/in_memory.rs diff --git a/crates/freenet2-node/src/lib.rs b/crates/locutus-node/src/lib.rs similarity index 86% rename from crates/freenet2-node/src/lib.rs rename to crates/locutus-node/src/lib.rs index 32885ed08..24f95be68 100644 --- a/crates/freenet2-node/src/lib.rs +++ b/crates/locutus-node/src/lib.rs @@ -7,7 +7,6 @@ mod operations; mod ring; mod user_events; -pub use conn_manager::PeerKey; pub use node::NodeConfig; type StdResult = std::result::Result; diff --git a/crates/freenet2-node/src/message.rs b/crates/locutus-node/src/message.rs similarity index 98% rename from crates/freenet2-node/src/message.rs rename to crates/locutus-node/src/message.rs index fecabc1ef..67f1a64ea 100644 --- a/crates/freenet2-node/src/message.rs +++ b/crates/locutus-node/src/message.rs @@ -1,3 +1,5 @@ +//! Main message type which encapsulated all the messaging between nodes. + use std::{fmt::Display, time::Duration}; use serde::{Deserialize, Serialize}; diff --git a/crates/freenet2-node/src/node.rs b/crates/locutus-node/src/node.rs similarity index 94% rename from crates/freenet2-node/src/node.rs rename to crates/locutus-node/src/node.rs index 8221ad07e..c4211d3b7 100644 --- a/crates/freenet2-node/src/node.rs +++ b/crates/locutus-node/src/node.rs @@ -1,3 +1,12 @@ +//! The main node data type which encapsulates all the behaviour for maintaining a connection +//! and performing operations within the network. +//! +//! # Implementations +//! Node comes with different underlying implementations that can be used upon construction. +//! Those implementations are: +//! - libp2p: all the connection is handled by libp2p. +//! - In memory: a simplifying node used for emulation pourpouses mainly. + use std::net::IpAddr; use libp2p::{identity, multiaddr::Protocol, Multiaddr, PeerId}; @@ -192,14 +201,7 @@ pub mod test_utils { use rand::Rng; use tokio::sync::mpsc; - use crate::{ - conn_manager::{ConnectionBridge, Transport}, - message::Message, - node::{InitPeerNode, NodeInMemory}, - operations::{join_ring::join_ring_op, OpError}, - ring::Distance, - NodeConfig, PeerKey, - }; + use crate::{NodeConfig, conn_manager::{ConnectionBridge, PeerKey, Transport}, message::Message, node::{InitPeerNode, NodeInMemory}, operations::{join_ring::join_ring_op, OpError}, ring::Distance}; pub fn get_free_port() -> Result { let mut port; @@ -228,7 +230,7 @@ pub mod test_utils { } pub(crate) struct NetEvent { - pub(crate) sender: String, + pub(crate) _sender: String, pub(crate) event: EventType, } diff --git a/crates/freenet2-node/src/node/in_memory.rs b/crates/locutus-node/src/node/in_memory.rs similarity index 98% rename from crates/freenet2-node/src/node/in_memory.rs rename to crates/locutus-node/src/node/in_memory.rs index 392c2113b..e45eef1d2 100644 --- a/crates/freenet2-node/src/node/in_memory.rs +++ b/crates/locutus-node/src/node/in_memory.rs @@ -1,9 +1,9 @@ use crate::{ - conn_manager::{in_memory::MemoryConnManager, ConnectionBridge, PeerKeyLocation}, + conn_manager::{in_memory::MemoryConnManager, ConnectionBridge, PeerKey, PeerKeyLocation}, message::Message, operations::{get, join_ring, put}, user_events::{test_utils::MemoryEventsGen, UserEvent, UserEventsProxy}, - NodeConfig, PeerKey, + NodeConfig, }; use super::op_state::OpStateStorage; diff --git a/crates/freenet2-node/src/node/libp2p_impl.rs b/crates/locutus-node/src/node/libp2p_impl.rs similarity index 100% rename from crates/freenet2-node/src/node/libp2p_impl.rs rename to crates/locutus-node/src/node/libp2p_impl.rs diff --git a/crates/freenet2-node/src/node/op_state.rs b/crates/locutus-node/src/node/op_state.rs similarity index 100% rename from crates/freenet2-node/src/node/op_state.rs rename to crates/locutus-node/src/node/op_state.rs diff --git a/crates/freenet2-node/src/operations.rs b/crates/locutus-node/src/operations.rs similarity index 100% rename from crates/freenet2-node/src/operations.rs rename to crates/locutus-node/src/operations.rs diff --git a/crates/freenet2-node/src/operations/get.rs b/crates/locutus-node/src/operations/get.rs similarity index 100% rename from crates/freenet2-node/src/operations/get.rs rename to crates/locutus-node/src/operations/get.rs diff --git a/crates/freenet2-node/src/operations/join_ring.rs b/crates/locutus-node/src/operations/join_ring.rs similarity index 98% rename from crates/freenet2-node/src/operations/join_ring.rs rename to crates/locutus-node/src/operations/join_ring.rs index 981acbca5..e11dbde94 100644 --- a/crates/freenet2-node/src/operations/join_ring.rs +++ b/crates/locutus-node/src/operations/join_ring.rs @@ -4,12 +4,11 @@ use rust_fsm::*; use super::{OpError, OperationResult}; use crate::{ - conn_manager::{self, ConnectionBridge, PeerKeyLocation}, + conn_manager::{self, ConnectionBridge, PeerKey, PeerKeyLocation}, message::{Message, Transaction, TransactionType}, node::{OpExecutionError, OpStateStorage}, operations::Operation, ring::{Location, Ring}, - PeerKey, }; pub(crate) use self::messages::{JoinRequest, JoinResponse, JoinRingMsg}; @@ -131,8 +130,8 @@ impl JRState { } } -/// Join ring routine, called upon processing a request to join or performing -/// a join operation for this node. +/// Join ring routine, called upon processing a request to join or while performing +/// a join operation for this node after initial request (see [`initial_join_request`]). /// /// # Cancellation Safety /// This future is not cancellation safe. @@ -403,6 +402,7 @@ where }) } +/// Join ring routine, called upon performing a join operation for this node. pub(crate) async fn initial_join_request( op_storage: &mut OpStateStorage, conn_manager: &mut CB, @@ -516,7 +516,7 @@ where mod messages { use super::*; - use crate::{conn_manager::PeerKeyLocation, ring::Location, PeerKey}; + use crate::{conn_manager::PeerKeyLocation, ring::Location}; use serde::{Deserialize, Serialize}; @@ -595,7 +595,6 @@ mod tests { config::tracing::Logger, message::TransactionTypeId, node::test_utils::{EventType, SimNetwork}, - PeerKey, }; #[test] diff --git a/crates/freenet2-node/src/operations/put.rs b/crates/locutus-node/src/operations/put.rs similarity index 100% rename from crates/freenet2-node/src/operations/put.rs rename to crates/locutus-node/src/operations/put.rs diff --git a/crates/freenet2-node/src/operations/subscribe.rs b/crates/locutus-node/src/operations/subscribe.rs similarity index 100% rename from crates/freenet2-node/src/operations/subscribe.rs rename to crates/locutus-node/src/operations/subscribe.rs diff --git a/crates/freenet2-node/src/probe_proto.rs b/crates/locutus-node/src/probe_proto.rs similarity index 100% rename from crates/freenet2-node/src/probe_proto.rs rename to crates/locutus-node/src/probe_proto.rs diff --git a/crates/freenet2-node/src/ring.rs b/crates/locutus-node/src/ring.rs similarity index 99% rename from crates/freenet2-node/src/ring.rs rename to crates/locutus-node/src/ring.rs index 56a02143d..b681edab4 100644 --- a/crates/freenet2-node/src/ring.rs +++ b/crates/locutus-node/src/ring.rs @@ -186,9 +186,8 @@ pub(crate) enum RingProtoError { mod tests { use libp2p::PeerId; - use crate::PeerKey; - use super::*; + use crate::conn_manager::PeerKey; #[test] fn location_dist() { diff --git a/crates/freenet2-node/src/user_events.rs b/crates/locutus-node/src/user_events.rs similarity index 100% rename from crates/freenet2-node/src/user_events.rs rename to crates/locutus-node/src/user_events.rs diff --git a/crates/freenet2-node/tests/node_startup.rs b/crates/locutus-node/tests/node_startup.rs similarity index 100% rename from crates/freenet2-node/tests/node_startup.rs rename to crates/locutus-node/tests/node_startup.rs From 290804b547138ed9d3ec952ff14fbff917df6264 Mon Sep 17 00:00:00 2001 From: Ignacio Duart Date: Sun, 26 Sep 2021 11:44:16 +0200 Subject: [PATCH 14/14] Minor fixes for join ring op --- .../src/conn_manager/in_memory.rs | 6 +- crates/locutus-node/src/message.rs | 3 +- crates/locutus-node/src/node.rs | 55 ++++++++++++++----- crates/locutus-node/src/node/in_memory.rs | 43 ++++++++------- .../locutus-node/src/operations/join_ring.rs | 33 +++++++---- 5 files changed, 90 insertions(+), 50 deletions(-) diff --git a/crates/locutus-node/src/conn_manager/in_memory.rs b/crates/locutus-node/src/conn_manager/in_memory.rs index 7182e7ea3..f8f2e5b26 100644 --- a/crates/locutus-node/src/conn_manager/in_memory.rs +++ b/crates/locutus-node/src/conn_manager/in_memory.rs @@ -41,7 +41,7 @@ impl MemoryConnManager { std::mem::drop(queue); } } - tokio::time::sleep(Duration::from_nanos(1000)).await; + tokio::time::sleep(Duration::from_millis(10)).await; } }); @@ -62,7 +62,7 @@ impl ConnectionBridge for MemoryConnManager { } std::mem::drop(queue); } - tokio::time::sleep(Duration::from_nanos(1000)).await; + tokio::time::sleep(Duration::from_millis(10)).await; } } @@ -120,7 +120,7 @@ impl InMemoryTransport { } Err(channel::TryRecvError::Disconnected) => break, Err(channel::TryRecvError::Empty) | Ok(_) => { - tokio::time::sleep(Duration::from_millis(1)).await + tokio::time::sleep(Duration::from_millis(10)).await } } } diff --git a/crates/locutus-node/src/message.rs b/crates/locutus-node/src/message.rs index 67f1a64ea..33d1df46b 100644 --- a/crates/locutus-node/src/message.rs +++ b/crates/locutus-node/src/message.rs @@ -103,8 +103,7 @@ mod sealed_msg_type { transaction_type_enumeration!(decl struct { JoinRing -> JoinRingMsg, Put -> PutMsg, - Get -> GetMsg, - Canceled -> Transaction + Get -> GetMsg }); } diff --git a/crates/locutus-node/src/node.rs b/crates/locutus-node/src/node.rs index c4211d3b7..07f5af5d0 100644 --- a/crates/locutus-node/src/node.rs +++ b/crates/locutus-node/src/node.rs @@ -11,7 +11,7 @@ use std::net::IpAddr; use libp2p::{identity, multiaddr::Protocol, Multiaddr, PeerId}; -use crate::config::CONF; +use crate::{config::CONF, ring::Location}; use self::libp2p_impl::NodeLibP2P; pub(crate) use in_memory::NodeInMemory; @@ -37,6 +37,16 @@ enum NodeImpl { InMemory(Box), } +/// When instancing a node you can either join an existing network or bootstrap a new network with a listener +/// which will act as the initial provider. This initial peer will be listening at the provided port and assigned IP. +/// If those are not free the instancing process will return an error. +/// +/// In order to bootstrap a new network the following arguments are required to be provided to the builder: +/// - ip: IP associated to the initial node. +/// - port: listening port of the initial node. +/// +/// If both are provided but also additional peers are added via the [`Self::add_provider()`] method, this node will +/// be listening but also try to connect to an existing peer. pub struct NodeConfig { /// local peer private key in local_key: identity::Keypair, @@ -50,19 +60,12 @@ pub struct NodeConfig { /// At least an other running listener node is required for joining the network. /// Not necessary if this is an initial node. remote_nodes: Vec, + + /// the location of this node, used for gateways. + location: Option, } impl NodeConfig { - /// When instancing a node you can either join an existing network or bootstrap a new network with a listener - /// which will act as the initial provider. This initial peer will be listening at the provided port and assigned IP. - /// If those are not free the instancing process will return an error. - /// - /// In order to bootstrap a new network the following arguments are required to be provided to the builder: - /// - ip: IP associated to the initial node. - /// - port: listening port of the initial node. - /// - /// If both are provided but also additional peers are added via the [add_provider] method, this node will - /// be listening but also try to connect to an existing peer. pub fn new() -> NodeConfig { let local_key = if let Some(key) = &CONF.local_peer_keypair { key.clone() @@ -74,6 +77,7 @@ impl NodeConfig { remote_nodes: Vec::with_capacity(1), local_ip: None, local_port: None, + location: None, } } @@ -94,6 +98,11 @@ impl NodeConfig { self } + pub fn with_location(mut self, loc: Location) -> Self { + self.location = Some(loc); + self + } + /// Connection info for an already existing peer. Required in case this is not a bootstrapping node. pub fn add_provider(mut self, peer: InitPeerNode) -> Self { self.remote_nodes.push(peer); @@ -123,6 +132,7 @@ impl Default for NodeConfig { pub struct InitPeerNode { addr: Option, identifier: Option, + location: Option, } impl InitPeerNode { @@ -130,6 +140,7 @@ impl InitPeerNode { Self { addr: None, identifier: None, + location: None, } } @@ -169,6 +180,11 @@ impl InitPeerNode { self.identifier = Some(id); self } + + pub fn with_location(mut self, loc: Location) -> Self { + self.location = Some(loc); + self + } } impl std::default::Default for InitPeerNode { @@ -180,6 +196,7 @@ impl std::default::Default for InitPeerNode { Self { addr: Some(multi_addr), identifier: Some(identifier), + location: None, } } } @@ -201,7 +218,14 @@ pub mod test_utils { use rand::Rng; use tokio::sync::mpsc; - use crate::{NodeConfig, conn_manager::{ConnectionBridge, PeerKey, Transport}, message::Message, node::{InitPeerNode, NodeInMemory}, operations::{join_ring::join_ring_op, OpError}, ring::Distance}; + use crate::{ + conn_manager::{ConnectionBridge, PeerKey, Transport}, + message::Message, + node::{InitPeerNode, NodeInMemory}, + operations::{join_ring::join_ring_op, OpError}, + ring::{Distance, Location}, + NodeConfig, + }; pub fn get_free_port() -> Result { let mut port; @@ -252,10 +276,12 @@ pub mod test_utils { let gateway_pair = identity::Keypair::generate_ed25519(); let gateway_peer_id = gateway_pair.public().into_peer_id(); let gateway_port = get_free_port().unwrap(); + let gateway_loc = Location::random(); let config = NodeConfig::new() .with_ip(Ipv6Addr::LOCALHOST) .with_port(gateway_port) - .with_key(gateway_pair); + .with_key(gateway_pair) + .with_location(gateway_loc); let gateway = NodeInMemory::build(config).unwrap(); sim.initialize_gateway(gateway, "gateway".to_owned()); @@ -266,7 +292,8 @@ pub mod test_utils { InitPeerNode::new() .listening_ip(Ipv6Addr::LOCALHOST) .listening_port(gateway_port) - .with_identifier(gateway_peer_id), + .with_identifier(gateway_peer_id) + .with_location(gateway_loc), ); sim.initialize_peer(NodeInMemory::build(config).unwrap(), label); } diff --git a/crates/locutus-node/src/node/in_memory.rs b/crates/locutus-node/src/node/in_memory.rs index e45eef1d2..995bb48aa 100644 --- a/crates/locutus-node/src/node/in_memory.rs +++ b/crates/locutus-node/src/node/in_memory.rs @@ -6,14 +6,13 @@ use crate::{ NodeConfig, }; -use super::op_state::OpStateStorage; +use super::{op_state::OpStateStorage, InitPeerNode}; pub(crate) struct NodeInMemory where Ev: UserEventsProxy, { peer: PeerKey, - listening: bool, gateways: Vec, pub conn_manager: MemoryConnManager, pub op_storage: OpStateStorage, @@ -29,49 +28,51 @@ impl NodeInMemory { } let peer = PeerKey::from(config.local_key.public()); let conn_manager = MemoryConnManager::new(true, peer, None); + let gateways = config + .remote_nodes + .into_iter() + .filter_map(|node| { + let InitPeerNode { + identifier, + location, + .. + } = node; + location.zip(identifier).map(|(loc, id)| PeerKeyLocation { + peer: PeerKey(id), + location: Some(loc), + }) + }) + .collect(); Ok(NodeInMemory { peer, - listening: true, conn_manager, op_storage: OpStateStorage::new(), - gateways: vec![], + gateways, user_events: MemoryEventsGen {}, }) } pub async fn start(&mut self) -> Result<(), ()> { - if self.listening { - // cannot start if already listening; meaning that this node already joined - // the ring or is a gateway - return Err(()); - } - // FIXME: this iteration should be shuffled, must write an extension iterator shuffle items "in place" // the idea here is to limit the amount of gateways being contacted that's why shuffling is required for gateway in &self.gateways { // initiate join action action per each gateway - let op = join_ring::JoinRingOp::new( - PeerKeyLocation { - peer: self.peer, - location: None, - }, + let op = join_ring::JoinRingOp::initial_request( + self.peer, *gateway, + self.op_storage.ring.max_hops_to_live, ); join_ring::initial_join_request(&mut self.op_storage, &mut self.conn_manager, op) .await .unwrap(); } - - Err(()) + Ok(()) } /// Starts listening to incoming messages, only allowed to be called directly when this node /// already joined the network. pub async fn listen_on(&mut self) -> Result<(), ()> { - if !self.listening { - return Err(()); - } - + self.start().await?; loop { tokio::select! { msg = self.conn_manager.recv() => { diff --git a/crates/locutus-node/src/operations/join_ring.rs b/crates/locutus-node/src/operations/join_ring.rs index e11dbde94..d0e8d889d 100644 --- a/crates/locutus-node/src/operations/join_ring.rs +++ b/crates/locutus-node/src/operations/join_ring.rs @@ -16,8 +16,24 @@ pub(crate) use self::messages::{JoinRequest, JoinResponse, JoinRingMsg}; pub(crate) struct JoinRingOp(StateMachine); impl JoinRingOp { - pub fn new(this_peer: PeerKeyLocation, gateway: PeerKeyLocation) -> Self { - JoinRingOp(StateMachine::new()) + pub fn initial_request( + req_peer: PeerKey, + target_loc: PeerKeyLocation, + max_hops_to_live: usize, + ) -> Self { + let mut sm = StateMachine::new(); + sm.consume(&JoinRingMsg::Req { + id: Transaction::new(::tx_type_id()), + msg: JoinRequest::Initial { + req_peer, + target_loc, + max_hops_to_live, + // initially is the max hops, will be decreased over each hop + hops_to_live: max_hops_to_live, + }, + }) + .unwrap(); + JoinRingOp(sm) } } @@ -418,7 +434,7 @@ where } = (&join_op.0).state().clone().try_unwrap_connecting()?; log::info!( - "Joining ring via {} at {}", + "Joining ring via {} (@{})", gateway.peer, gateway .location @@ -442,10 +458,6 @@ where gateway.peer ); conn_manager.send(&gateway, join_req).await?; - // join_op - // .0 - // .consume(&JoinRingMsg::Req { , this_peer }) - // .map_err(|_| OpError::IllegalStateTransition)?; op_storage.push(tx, Operation::JoinRing(join_op))?; Ok(()) } @@ -653,12 +665,13 @@ mod tests { assert!(matches!(join_op_host_2.state(), JRState::Connected)); } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + // #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn node0_to_gateway_conn() -> Result<(), Box> { //! Given a network of one node and one gateway test that both are connected. Logger::init_logger(); let mut sim_net = SimNetwork::build(1, 1, 0); - match tokio::time::timeout(Duration::from_secs(1), sim_net.recv_net_events()).await { + tokio::time::sleep(Duration::from_secs(300)).await; + match tokio::time::timeout(Duration::from_secs(300), sim_net.recv_net_events()).await { Ok(Some(Ok(event))) => match event.event { EventType::JoinSuccess { gateway, new_node } => { log::info!("Successful join op between {} and {}", gateway, new_node); @@ -675,7 +688,7 @@ mod tests { Logger::init_logger(); let _sim_nodes = SimNetwork::build(10, 10, 7); - tokio::time::sleep(Duration::from_secs(300)).await; + // tokio::time::sleep(Duration::from_secs(300)).await; // let _hist: Vec<_> = _ring_distribution(sim_nodes.values()).collect(); // FIXME: enable probing