Added publicly available network map

fixes: bosagora#251

.github/workflows/main.yml

@@ -65,7 +65,7 @@ jobs:
       if: runner.os == 'Linux'
       run: |
         sudo apt-get update
-        sudo apt-get install libsodium-dev libsqlite3-dev clang
+        sudo apt-get install libsodium-dev libsqlite3-dev clang libmaxminddb-dev
 
     - name: '[Windows] Install dependencies & setup environment'
       if: runner.os == 'Windows'

Dockerfile

@@ -19,7 +19,7 @@ COPY devel/dotgdbinit /root/.gdbinit
 COPY --from=Builder /root/packages/ /root/packages/
 RUN apk --no-cache add --allow-untrusted -X /root/packages/build/ ldc-runtime=1.26.0-r0 \
     && rm -rf /root/packages/
-RUN apk --no-cache add llvm-libunwind libgcc libsodium libstdc++ sqlite-libs
+RUN apk --no-cache add llvm-libunwind libgcc libsodium libstdc++ sqlite-libs libmaxminddb
 COPY --from=Builder /root/agora/talos/build/ /usr/share/agora/talos/
 COPY --from=Builder /root/agora/build/agora /usr/local/bin/agora
 WORKDIR /agora/

doc/config.example.yaml

@@ -55,6 +55,20 @@ node:
   # after `relay_tx_cache_exp_secs`.
   relay_tx_cache_exp_secs : 1200
 
+  # true, if this node should collect statistics about other
+  # nodes in the network, including their geographical location and OS
+  collect_network_statistics : true
+
+  # The number of network crawlers that will be instantiated to collects
+  # statistics about other nodes in the network
+  num_of_crawlers : 3
+
+  # The number of seconds one crawler should wait after successfully contacted a node
+  crawling_interval_secs : 2
+
+  # The path to a file containing IP address -> geographical location mapping
+  ipdb_path : "data/ipdb.mmdb"
+
 # Each entry in this array is an interface Agora will listen to, allowing to
 # expose the same node on more than one network interface or with different
 # API, such as having one interface using HTTP+JSON and the other TCP+binary.

dub.json

@@ -69,6 +69,7 @@
     "dflags": [ "-extern-std=c++14", "-preview=in" ],
     "lflags-posix": [ "-lstdc++", "-lsqlite3" ],
     "lflags-windows": [  "sqlite3.lib", "/nodefaultlib:msvcetd.lib" ],
+    "libs": [ "maxminddb" ],
     "libs-windows": [ "iphlpapi" ],
     "buildRequirements": [ "allowWarnings" ],
 

source/agora/api/FullNode.d

@@ -21,6 +21,7 @@ import agora.consensus.data.PreImageInfo;
 import agora.common.Types;
 import agora.common.Set;
 import agora.consensus.data.Transaction;
+import agora.network.Crawler : CrawlResultHolder;
 
 import vibe.data.serialization;
 import vibe.http.common;
@@ -41,6 +42,9 @@ public struct NodeInfo
 
     /// Partial or full view of the addresses of the node's quorum (based on is_complete)
     public Set!string addresses;
+
+    /// String representation of the OS the node is running on
+    public string os;
 }
 
 /*******************************************************************************
@@ -227,6 +231,23 @@ public interface API
 
     public PreImageInfo getPreimage (Hash enroll_key);
 
+    /***************************************************************************
+
+        Get the information about the network and it's reachable nodes.
+
+        The information includes the network address and the geograpical location
+        of the nodes including the continent/country/city/latitude/longitude.
+
+        API:
+            GET /network_info
+
+        Returns: information about the network and about it's nodes
+
+
+    ***************************************************************************/
+
+    public CrawlResultHolder getNetworkInfo ();
+
     /***************************************************************************
 
         Get validators' pre-image information

source/agora/common/Config.d

@@ -228,6 +228,21 @@ public struct NodeConfig
     /// Transaction put into the relay queue will expire, and will be removed
     /// after `relay_tx_cache_exp`.
     public Duration relay_tx_cache_exp;
+
+    /// true, if this node should collect statistics about other
+    /// nodes in the network, including their geographical location and OS
+    public bool collect_network_statistics;
+
+    /// The number of network crawlers that will be instantiated to collects
+    /// statistics about other nodes in the network
+    public ubyte num_of_crawlers;
+
+    /// The number of seconds one crawler should wait after successfully contacted
+    /// a node
+    public Duration crawling_interval;
+
+    /// The path to a file containing IP address -> geographical location mapping
+    public string ipdb_path;
 }
 
 /// Validator config
@@ -524,6 +539,10 @@ private NodeConfig parseNodeConfig (Node* node, in CommandLine cmdln)
     Duration relay_tx_interval = opt!(ulong, "node", "relay_tx_interval_secs")(cmdln, node, 15).seconds;
     const relay_tx_min_fee = Amount(opt!(ulong, "node", "relay_tx_min_fee")(cmdln, node, 0));
     Duration relay_tx_cache_exp = opt!(ulong, "node", "relay_tx_cache_exp_secs")(cmdln, node, 1200).seconds;
+    const collect_network_statistics = opt!(bool, "node", "collect_network_statistics")(cmdln, node, true);
+    const num_of_crawlers = opt!(ubyte, "node", "num_of_crawlers")(cmdln, node, 1);
+    Duration crawling_interval = opt!(ulong, "node", "crawling_interval_secs")(cmdln, node, 3).seconds;
+    const ipdb_path = opt!(string, "node", "ipdb_path")(cmdln, node, "data/ipdb.mmdb");
 
     NodeConfig result = {
             min_listeners : min_listeners,
@@ -545,6 +564,10 @@ private NodeConfig parseNodeConfig (Node* node, in CommandLine cmdln)
             relay_tx_interval : relay_tx_interval,
             relay_tx_min_fee : relay_tx_min_fee,
             relay_tx_cache_exp : relay_tx_cache_exp,
+            collect_network_statistics : collect_network_statistics,
+            num_of_crawlers : num_of_crawlers,
+            crawling_interval : crawling_interval,
+            ipdb_path : ipdb_path,
     };
     return result;
 }