Supported new multi-networks API of DHT. United crate for protocols

CHANGELOG.md

@@ -2,6 +2,11 @@
 
 All notable changes to this project will be documented in this file.
 
+## Version 0.56.4
+
+- Supported new multi-networks API of DHT
+- United crate for protocols
+
 ## Version 0.56.3
 
 - Fixed UNREGISTERED_CHAIN_MAX_LEN const for Venom blockchain.

Cargo.toml

@@ -2,7 +2,7 @@
 build = 'common/build/build.rs'
 edition = '2021'
 name = 'ton_node'
-version = '0.56.3'
+version = '0.56.4'
 
 [workspace]
 members = [ 'storage' ]
@@ -84,20 +84,17 @@ stream-cancel = '0.8.0'
 string-builder = '^0.2.0'
 tokio = { features = [ 'rt-multi-thread' ], version = '1.5' }
 tokio-util = '0.7'
-adnl = { features = [ 'client', 'node', 'server' ], git = 'https://github.com/tonlabs/ever-adnl.git', tag = '0.10.2' }
+adnl = { features = [ 'client', 'node', 'server' ], git = 'https://github.com/tonlabs/ever-adnl.git', tag = '0.10.12' }
 catchain = { path = 'catchain' }
-dht = { git = 'https://github.com/tonlabs/ever-dht.git', tag = '0.6.96' }
 lockfree = { git = 'https://github.com/tonlabs/lockfree.git' }
-overlay = { git = 'https://github.com/tonlabs/ever-overlay.git', tag = '0.7.27' }
-rldp = { git = 'https://github.com/tonlabs/ever-rldp.git', tag = '0.8.21' }
 storage = { path = 'storage' }
-ton_abi = { git = 'https://github.com/tonlabs/ever-abi.git', optional = true, tag = '2.4.16' }
-ton_api = { git = 'https://github.com/tonlabs/ever-tl.git', package = 'ton_api', tag = '0.3.62' }
-ton_block = { git = 'https://github.com/tonlabs/ever-block.git', tag = '1.9.126' }
-ton_block_json = { git = 'https://github.com/tonlabs/ever-block-json.git', tag = '0.7.215' }
-ton_executor = { git = 'https://github.com/tonlabs/ever-executor.git', tag = '1.16.112' }
-ton_types = { git = 'https://github.com/tonlabs/ever-types.git', tag = '2.0.33' }
-ton_vm = { git = 'https://github.com/tonlabs/ever-vm.git', tag = '1.9.9' }
+ton_abi = { git = 'https://github.com/tonlabs/ever-abi.git', optional = true, tag = '2.4.23' }
+ton_api = { git = 'https://github.com/tonlabs/ever-tl.git', package = 'ton_api', tag = '0.3.69' }
+ton_block = { git = 'https://github.com/tonlabs/ever-block.git', tag = '1.9.139' }
+ton_block_json = { git = 'https://github.com/tonlabs/ever-block-json.git', tag = '0.7.228' }
+ton_executor = { git = 'https://github.com/tonlabs/ever-executor.git', tag = '1.16.119' }
+ton_types = { git = 'https://github.com/tonlabs/ever-types.git', tag = '2.0.38' }
+ton_vm = { git = 'https://github.com/tonlabs/ever-vm.git', tag = '1.9.20' }
 validator_session = { path = 'validator-session' }
 
 [dev-dependencies]
@@ -118,7 +115,7 @@ prometheus = [ 'metrics-exporter-prometheus', 'log_metrics' ]
 signature_with_id = [ 'ton_block/signature_with_id', 'ton_vm/signature_with_id', 'ton_executor/signature_with_id' ]
 slashing = [ 'ton_abi', 'validator_session/slashing' ]
 statsd = [ 'metrics-exporter-statsd', 'log_metrics', 'dep:statsd' ]
-telemetry = [ 'adnl/telemetry', 'dht/telemetry', 'rldp/telemetry', 'overlay/telemetry', 'storage/telemetry' ]
+telemetry = [ 'adnl/telemetry', 'storage/telemetry' ]
 trace_alloc = [  ]
 trace_alloc_detail = [ 'trace_alloc' ]
 fast_finality_extra = [  ]
@@ -133,3 +130,4 @@ version = '0.26.0'
 features = [ 'cmake_build' ]
 optional = true
 version = '0.26.0'
+

bin/adnl_ping.rs

@@ -15,7 +15,7 @@ use adnl::{
     common::TaggedTlObject, 
     node::{AdnlNode, AdnlNodeConfig, AdnlNodeConfigJson, IpAddress}
 };
-use overlay::OverlayNode;
+use adnl::OverlayNode;
 use std::{convert::TryInto, env, fs::File, io::BufReader, sync::Arc};
 use ton_api::ton::{TLObject, rpc::ton_node::GetCapabilities};
 #[cfg(feature = "telemetry")]

bin/adnl_resolve.rs

@@ -12,7 +12,7 @@
 */
 
 use adnl::node::{AdnlNode, AdnlNodeConfig};
-use dht::{DhtNode, DhtSearchPolicy};
+use adnl::{DhtNode, DhtSearchPolicy};
 use std::{convert::TryInto, env, fs::File, io::BufReader};
 use ton_node::config::TonNodeGlobalConfigJson;
 use ton_types::{error, fail, base64_decode, KeyId, Result};
@@ -35,13 +35,13 @@ async fn scan(adnlid: &str, cfgfile: &str) -> Result<()> {
         vec![KEY_TAG]
     )?;
     let adnl = AdnlNode::with_config(config).await?;
-    let dht = DhtNode::with_adnl_node(adnl.clone(), KEY_TAG)?;
+    let dht = DhtNode::with_params(adnl.clone(), KEY_TAG, None)?;
     AdnlNode::start(&adnl, vec![dht.clone()]).await?;
 
     let mut nodes = Vec::new();
     let mut bad_nodes = Vec::new();
     for dht_node in dht_nodes.iter() {
-        if let Some(key) = dht.add_peer(dht_node)? {
+        if let Some(key) = dht.add_peer_to_network(dht_node, None)? {
             nodes.push(key)
         } else {
             fail!("Invalid DHT peer {:?}", dht_node)
@@ -53,19 +53,20 @@ async fn scan(adnlid: &str, cfgfile: &str) -> Result<()> {
     let mut index = 0;
     println!("Searching DHT for {}...", keyid);
     loop {
-        if let Ok(Some((ip, key))) = DhtNode::find_address_with_context(
+        if let Ok(Some((ip, key))) = DhtNode::find_address_in_network_with_context(
             &dht, 
             &keyid,
             &mut context,
-            DhtSearchPolicy::FastSearch(5)
+            DhtSearchPolicy::FastSearch(5),
+            None
         ).await {
             println!("Found {} / {}", ip, key.id());
             return Ok(())
         }
         if index >= nodes.len() {
             nodes.clear();
-            for dht_node in dht.get_known_nodes(10000)?.iter() {
-                if let Some(key) = dht.add_peer(dht_node)? {
+            for dht_node in dht.get_known_nodes_of_network(10000, None)?.iter() {
+                if let Some(key) = dht.add_peer_to_network(dht_node, None)? {
                     if !bad_nodes.contains(&key) {
                         nodes.push(key)
                     }
@@ -82,14 +83,14 @@ async fn scan(adnlid: &str, cfgfile: &str) -> Result<()> {
             index, 
             nodes.len()
         );
-        if !dht.find_dht_nodes(&nodes[index]).await? {
+        if !dht.find_dht_nodes_in_network(&nodes[index], None).await? {
             println!("DHT node {} is non-responsive", nodes[index]); 
             bad_nodes.push(nodes.remove(index))
         } else {
             index += 1
         }
     }
-
+    
 }
 
 #[tokio::main]

bin/dhtscan.rs

@@ -12,8 +12,8 @@
 */
 
 use adnl::node::{AdnlNode, AdnlNodeConfig};
-use dht::DhtNode;
-use overlay::OverlayNode;
+use adnl::DhtNode;
+use adnl::OverlayNode;
 use std::{collections::HashMap, env, fs::File, io::BufReader, ops::Deref, sync::Arc};
 use ton_node::config::TonNodeGlobalConfigJson;
 use ton_types::{error, fail, base64_encode, KeyOption, Result};
@@ -40,7 +40,7 @@ fn scan(cfgfile: &str, jsonl: bool, search_overlay: bool, use_workchain0: bool)
         vec![KEY_TAG]
     )?;
     let adnl = rt.block_on(AdnlNode::with_config(config))?;
-    let dht = DhtNode::with_adnl_node(adnl.clone(), KEY_TAG)?;
+    let dht = DhtNode::with_params(adnl.clone(), KEY_TAG, None)?;
     let overlay = OverlayNode::with_adnl_node_and_zero_state(
         adnl.clone(), 
         zero_state.as_slice(), 
@@ -51,7 +51,7 @@ fn scan(cfgfile: &str, jsonl: bool, search_overlay: bool, use_workchain0: bool)
 
     let mut preset_nodes = Vec::new();
     for dht_node in dht_nodes.iter() {
-        if let Some(key) = dht.add_peer(dht_node)? {
+        if let Some(key) = dht.add_peer_to_network(dht_node, None)? {
             preset_nodes.push(key)
         } else {
             fail!("Invalid DHT peer {:?}", dht_node)
@@ -60,7 +60,7 @@ fn scan(cfgfile: &str, jsonl: bool, search_overlay: bool, use_workchain0: bool)
 
     println!("Scanning DHT...");
     for node in preset_nodes.iter() {
-        rt.block_on(dht.find_dht_nodes(node))?;
+        rt.block_on(dht.find_dht_nodes_in_network(node, None))?;
     }
 
     scan_overlay(&mut rt, &dht, preset_nodes.len(), &overlay, search_overlay, -1)?;
@@ -72,7 +72,7 @@ fn scan(cfgfile: &str, jsonl: bool, search_overlay: bool, use_workchain0: bool)
     }
 
     let mut count = 0;
-    let nodes = dht.get_known_nodes(5000)?;
+    let nodes = dht.get_known_nodes_of_network(5000, None)?;
     if nodes.len() > dht_nodes.len() {
         println!("---- Found DHT nodes:");
         for node in nodes {
@@ -87,7 +87,7 @@ fn scan(cfgfile: &str, jsonl: bool, search_overlay: bool, use_workchain0: bool)
                 continue;
             }
             let key: Arc<dyn KeyOption> = (&node.id).try_into()?;
-            match rt.block_on(dht.ping(key.id())) {
+            match rt.block_on(dht.ping_in_network(key.id(), None)) {
                 Ok(true) => (),
                 _ => continue
             }
@@ -151,7 +151,9 @@ fn scan_overlay(
     let mut iter = None;
     let mut overlays = HashMap::new();
     loop {
-        let res = rt.block_on(DhtNode::find_overlay_nodes(&dht, &overlay_id, &mut iter))?;
+        let res = rt.block_on(
+            DhtNode::find_overlay_nodes_in_network(&dht, &overlay_id, &mut iter, None)
+        )?;
         let count = overlays.len();
         for (ip, node) in res {
             let key: Arc<dyn KeyOption> = (&node.id).try_into()?;
@@ -165,7 +167,7 @@ fn scan_overlay(
         } else {
             println!(
                 "Found {} new DHT nodes, searching more...", 
-                dht.get_known_nodes(5000)?.len() - dht_presets
+                dht.get_known_nodes_of_network(5000, None)?.len() - dht_presets
             )
         }
         if iter.is_none() {

bin/gendht.rs

@@ -12,7 +12,7 @@
 */
 
 use adnl::{adnl_node_test_key, adnl_node_test_config, node::{AdnlNode, AdnlNodeConfig}};
-use dht::DhtNode;
+use adnl::DhtNode;
 use std::{env, ops::Deref, sync::Arc};
 use ton_types::{error, base64_encode, KeyOption, Result};
 
@@ -21,8 +21,8 @@ async fn gen(ip: &str, dht_key_enc: &str) -> Result<()> {
        adnl_node_test_config!(ip, adnl_node_test_key!(1 as usize, dht_key_enc))
     ).unwrap();
     let adnl = AdnlNode::with_config(config).await.unwrap();
-    let dht = DhtNode::with_adnl_node(adnl.clone(), 1 as usize).unwrap();
-    let node = dht.get_signed_node().unwrap();
+    let dht = DhtNode::with_params(adnl.clone(), 1 as usize, None).unwrap();
+    let node = dht.get_signed_node_of_network(None).unwrap();
     let key: Arc<dyn KeyOption> = (&node.id).try_into()?;
     let adr = AdnlNode::parse_address_list(&node.addr_list)?.ok_or_else(
         || error!("Cannot parse address list {:?}", node.addr_list)

catchain/Cargo.toml

@@ -19,12 +19,10 @@ quanta = '0.11.1'
 rand = '0.8'
 regex = '1.3.1'
 tokio = { features = [ 'rt-multi-thread' ], version = '1.5' }
-adnl = { features = [ 'node' ], git = 'https://github.com/tonlabs/ever-adnl.git', tag = '0.10.2' }
-overlay = { git = 'https://github.com/tonlabs/ever-overlay.git', tag = '0.7.27' }
-rldp = { git = 'https://github.com/tonlabs/ever-rldp.git', tag = '0.8.21' }
+adnl = { features = [ 'node' ], git = 'https://github.com/tonlabs/ever-adnl.git', tag = '0.10.12' }
 storage = { path = '../storage' }
-ton_api = { git = 'https://github.com/tonlabs/ever-tl.git', package = 'ton_api', tag = '0.3.62' }
-ton_types = { git = 'https://github.com/tonlabs/ever-types.git', tag = '2.0.33' }
+ton_api = { git = 'https://github.com/tonlabs/ever-tl.git', package = 'ton_api', tag = '0.3.69' }
+ton_types = { git = 'https://github.com/tonlabs/ever-types.git', tag = '2.0.38' }
 
 [dev-dependencies]
 chrono = '0.4.10'

catchain/src/catchain.rs

@@ -22,8 +22,8 @@ use crate::{
     ReceiverTaskQueuePtr, SessionId, ton, 
     profiling::Profiler, utils::{self, get_elapsed_time, MetricsDumper, MetricsHandle}
 };
+use adnl::{OverlayUtils, PrivateOverlayShortId};                    
 use metrics::Recorder;
-use overlay::{OverlayUtils, PrivateOverlayShortId};                    
 use rand::Rng;                                                              
 use std::{
     any::Any, cell::RefCell, collections::HashMap, collections::LinkedList, fmt, rc::Rc, 

catchain/src/lib.rs

@@ -26,7 +26,7 @@ pub mod utils;
 
 use crate::{profiling::InstanceCounter, utils::MetricsHandle};
 use adnl::node::AdnlNode;
-use overlay::PrivateOverlayShortId;
+use adnl::PrivateOverlayShortId;
 use std::{
     any::Any, cell::RefCell, fmt, path::Path, rc::{Rc, Weak}, sync::Arc,
     time::SystemTime,

catchain/src/log_player.rs

@@ -18,7 +18,7 @@ use crate::{
     ExternalQueryResponseCallback, LogPlayer, LogPlayerPtr, LogReplayOptions, Options, 
     PublicKeyHash, PrivateKey, SessionId, ValidatorWeight, utils
 };
-use overlay::PrivateOverlayShortId;
+use adnl::PrivateOverlayShortId;
 use std::{
     cell::RefCell, collections::HashMap, fmt, fs::File, io::{BufRead, BufReader}, 
     rc::Rc, sync::{Arc, atomic::{AtomicBool, Ordering}}, time::SystemTime

src/engine.rs

@@ -67,8 +67,8 @@ use crate::{
 
 #[cfg(feature = "telemetry")]
 use adnl::telemetry::{Metric, MetricBuilder, TelemetryItem, TelemetryPrinter};
+use adnl::QueriesConsumer;
 use catchain::SessionId;
-use overlay::QueriesConsumer;
 use std::{
     ops::Deref, sync::{Arc, atomic::{AtomicBool, AtomicU8, AtomicU32, Ordering, AtomicU64}},
     time::{Duration, SystemTime}, collections::{HashMap, HashSet}, path::Path

src/engine_operations.rs

@@ -43,10 +43,10 @@ use crate::{
 #[cfg(feature = "external_db")]
 use crate::internal_db::EXTERNAL_DB_BLOCK;
 
+use adnl::{BroadcastSendInfo, PrivateOverlayShortId};
 use catchain::{
     CatchainNode, CatchainOverlay, CatchainOverlayListenerPtr, CatchainOverlayLogReplayListenerPtr
 };
-use overlay::{BroadcastSendInfo, PrivateOverlayShortId};
 use std::{collections::HashSet, ops::Deref, sync::Arc};
 use storage::block_handle_db::BlockHandle;
 use ton_api::{

src/engine_traits.rs

@@ -31,13 +31,13 @@ use crate::{
 
 #[cfg(feature = "telemetry")]
 use adnl::telemetry::Metric;
+use adnl::{
+    BroadcastSendInfo, OverlayId, OverlayShortId, QueriesConsumer, PrivateOverlayShortId
+};
 use catchain::{
     CatchainNode, CatchainOverlay, CatchainOverlayListenerPtr, 
     CatchainOverlayLogReplayListenerPtr
 };
-use overlay::{
-    BroadcastSendInfo, OverlayId, OverlayShortId, QueriesConsumer, PrivateOverlayShortId
-};
 use std::{collections::HashSet, sync::{Arc, atomic::AtomicU64}};
 use storage::{StorageAlloc, block_handle_db::BlockHandle};
 #[cfg(feature = "telemetry")]

src/main.rs

@@ -285,9 +285,6 @@ fn get_build_info() -> String {
         Rust: {}\n\
         TON NODE git commit:         {}\n\
         ADNL git commit:             {}\n\
-        DHT git commit:              {}\n\
-        OVERLAY git commit:          {}\n\
-        RLDP git commit:             {}\n\
         TON_BLOCK git commit:        {}\n\
         TON_BLOCK_JSON git commit:   {}\n\
         TON_EXECUTOR git commit:     {}\n\
@@ -298,9 +295,6 @@ fn get_build_info() -> String {
         std::option_env!("BUILD_RUST_VERSION").unwrap_or(NOT_SET_LABEL),
         std::option_env!("BUILD_GIT_COMMIT").unwrap_or(NOT_SET_LABEL),
         adnl::build_commit().unwrap_or(NOT_SET_LABEL),
-        dht::build_commit().unwrap_or(NOT_SET_LABEL),
-        overlay::build_commit().unwrap_or(NOT_SET_LABEL),
-        rldp::build_commit().unwrap_or(NOT_SET_LABEL),
         ton_block::build_commit().unwrap_or(NOT_SET_LABEL),
         ton_block_json::build_commit().unwrap_or(NOT_SET_LABEL),
         ton_executor::build_commit().unwrap_or(NOT_SET_LABEL),

src/network/catchain_client.rs

@@ -21,12 +21,12 @@ use adnl::{
     },
     node::AdnlNode,
 };
+use adnl::{OverlayNode, PrivateOverlayShortId, QueriesConsumer};
+use adnl::RldpNode;
 use catchain::{
     BlockPayloadPtr, CatchainNode, CatchainOverlay, CatchainOverlayListenerPtr,
     ExternalQueryResponseCallback, PublicKeyHash
 };
-use overlay::{OverlayNode, PrivateOverlayShortId, QueriesConsumer};
-use rldp::RldpNode;
 use std::{
     collections::HashMap, io::Cursor, sync::{Arc, atomic::{self, AtomicBool}}, time::Instant
 };

src/network/full_node_client.rs

@@ -28,7 +28,7 @@ use adnl::{
     common::{CountedObject, Counter,TaggedByteSlice, TaggedObject, TaggedTlObject},
     node::AdnlNode
 };
-use overlay::{BroadcastSendInfo, OverlayShortId, OverlayNode};
+use adnl::{BroadcastSendInfo, OverlayShortId, OverlayNode};
 use rand::seq::SliceRandom;
 use std::{io::Cursor, time::Instant, sync::Arc, time::Duration, collections::HashSet};
 #[cfg(feature = "telemetry")]

src/network/full_node_service.rs

@@ -17,7 +17,7 @@ use crate::{
 };
 
 use adnl::common::{AdnlPeers, Answer, QueryAnswer, QueryResult, TaggedByteVec, TaggedObject};
-use overlay::QueriesConsumer;
+use adnl::QueriesConsumer;
 use std::{cmp::min, fmt::Debug, sync::Arc};
 #[cfg(feature = "telemetry")]
 use ton_api::{tag_from_boxed_type, tag_from_boxed_object};

src/network/neighbours.rs

@@ -14,8 +14,8 @@
 use crate::network::node_network::NodeNetwork;
 
 use adnl::{common::{Query, TaggedTlObject, Wait}, node::{AdnlNode, AddressCache}};
-use dht::DhtNode;
-use overlay::{OverlayShortId, OverlayNode};
+use adnl::DhtNode;
+use adnl::{OverlayShortId, OverlayNode};
 use rand::Rng;
 use std::{
     cmp::min,
@@ -456,7 +456,7 @@ impl Neighbours {
         tokio::spawn(async move {
             for peer in peers.iter() {
                 log::trace!("add_new_peers: searching IP for peer {}...", peer);
-                match DhtNode::find_address(&this.dht, peer).await {
+                match DhtNode::find_address_in_network(&this.dht, peer, None).await {
                     Ok(Some((ip, _))) => {
                         log::info!("add_new_peers: peer {}, IP {}", peer, ip);
                         if !this.add_overlay_peer(peer.clone()) {