Added publicly available network map

fixes: bosagora#251

.github/workflows/main.yml

@@ -45,6 +45,19 @@ jobs:
         submodules: true
         persist-credentials: false
 
+    # Agora unittests need infrastructure/bigfiles/ip.mmdb
+    - name: Checkout infrastructure
+      uses: actions/checkout@v2
+      with:
+        path: infrastructure
+        repository: bosagora/infrastructure
+        ref: master
+        submodules: false
+        lfs: true
+
+    - name: Checkout LFS objects
+      run: git lfs checkout
+
     # Install the D compiler
     - name: Prepare compiler
       uses: dlang-community/setup-dlang@v1
@@ -57,15 +70,15 @@ jobs:
     - name: '[OSX] Install dependencies & setup environment'
       if: runner.os == 'macOS'
       run: |
-        brew install coreutils libsodium pkg-config
+        brew install coreutils libsodium pkg-config libmaxminddb
         echo "LIBRARY_PATH=${LD_LIBRARY_PATH-}:/usr/local/lib/" >> $GITHUB_ENV
         echo "PKG_CONFIG_PATH=/usr/local/opt/sqlite/lib/pkgconfig:/usr/local/opt/openssl@1.1/lib/pkgconfig/" >> $GITHUB_ENV
 
     - name: '[Linux] Install dependencies & setup environment'
       if: runner.os == 'Linux'
       run: |
         sudo apt-get update
-        sudo apt-get install libsodium-dev libsqlite3-dev clang
+        sudo apt-get install libsodium-dev libsqlite3-dev clang libmaxminddb-dev
 
     - name: '[Windows] Install dependencies & setup environment'
       if: runner.os == 'Windows'

.gitignore

@@ -46,3 +46,7 @@ source/scpp/extra/DVMChecks.cpp
 
 # Agora log directory when Agora is started up with config.example.yml
 log/
+
+# Infrastructure project containing GeoIp
+infrastructure/
+

Dockerfile

@@ -2,7 +2,7 @@
 FROM bpfk/agora-builder:latest AS Builder
 ARG DUB_OPTIONS
 ARG AGORA_VERSION="HEAD"
-RUN apk --no-cache add linux-headers python3 npm
+RUN apk --no-cache add linux-headers python3 npm libmaxminddb-dev
 ADD . /root/agora/
 WORKDIR /root/agora/talos/
 RUN npm ci && npm run build
@@ -19,7 +19,7 @@ COPY devel/dotgdbinit /root/.gdbinit
 COPY --from=Builder /root/packages/ /root/packages/
 RUN apk --no-cache add --allow-untrusted -X /root/packages/build/ ldc-runtime=1.26.0-r0 \
     && rm -rf /root/packages/
-RUN apk --no-cache add llvm-libunwind libgcc libsodium libstdc++ sqlite-libs
+RUN apk --no-cache add llvm-libunwind libgcc libsodium libstdc++ sqlite-libs libmaxminddb-dev
 COPY --from=Builder /root/agora/talos/build/ /usr/share/agora/talos/
 COPY --from=Builder /root/agora/build/agora /usr/local/bin/agora
 WORKDIR /agora/

doc/config.example.yaml

@@ -58,6 +58,20 @@ node:
   # increased in order to be added to the transaction pool
   double_spent_threshold_pct: 20
 
+  # true, if this node should collect statistics about other
+  # nodes in the network, including their geographical location and OS
+  collect_network_statistics : true
+
+  # The number of network crawlers that will be instantiated to collect
+  # statistics about other nodes in the network
+  num_of_crawlers : 3
+
+  # The number of seconds one crawler should wait after successfully contacted a node
+  crawling_interval_secs : 2
+
+  # The path to a file containing IP address -> geographical location mapping
+  ipdb_path : "data/ipdb.mmdb"
+
 # Each entry in this array is an interface Agora will listen to, allowing to
 # expose the same node on more than one network interface or with different
 # API, such as having one interface using HTTP+JSON and the other TCP+binary.

dub.json

@@ -73,6 +73,7 @@
     "lflags-windows": [  "sqlite3.lib", "/nodefaultlib:msvcetd.lib" ],
     "lflags-linux": [ "--export-dynamic" ],
     "lflags-osx": [ "-export_dynamic" ],
+    "libs": [ "maxminddb" ],
     "libs-windows": [ "iphlpapi" ],
     "buildRequirements": [ "allowWarnings" ],
 
@@ -114,19 +115,6 @@
             "targetName": "agora-client",
             "mainSourceFile": "source/agora/cli/client/main.d",
             "excludedSourceFiles": [
-                "source/agora/api/Validator.d",
-                "source/agora/common/Config.d",
-                "source/agora/common/SCPHash.d",
-                "source/agora/consensus/Quorum.d",
-                "source/agora/network/Client.d",
-                "source/agora/network/Manager.d",
-                "source/agora/node/*",
-                "source/agora/flash/*",
-                "source/agora/utils/SCPPrettyPrinter.d",
-                "source/scpp/*",
-                "source/scpd/*",
-                "source/agora/consensus/protocol/Nominator.d",
-                "source/agora/consensus/SCPEnvelopeStore.d",
                 "source/agora/cli/checkvtable/*",
                 "source/agora/cli/multi/*",
                 "source/agora/cli/vanity/*",

source/agora/api/FullNode.d

@@ -21,6 +21,7 @@ import agora.consensus.data.PreImageInfo;
 import agora.common.Types;
 import agora.common.Set;
 import agora.consensus.data.Transaction;
+import agora.network.CrawlerTypes;
 
 import vibe.data.serialization;
 import vibe.http.common;
@@ -41,6 +42,21 @@ public struct NodeInfo
 
     /// Partial or full view of the addresses of the node's quorum (based on is_complete)
     public Set!string addresses;
+
+    /// String representation of the OS the node is running on
+    public string os;
+
+    /// true, if this node should be included in network statistics
+    public bool include_in_network_statistics;
+
+    /// Name of the client
+    public string client_name = "Agora";
+
+    /// Version of the client
+    public string client_ver;
+
+    /// Blockchain height known to the client
+    public Height height;
 }
 
 /*******************************************************************************
@@ -227,6 +243,23 @@ public interface API
 
     public PreImageInfo getPreimage (Hash enroll_key);
 
+    /***************************************************************************
+
+        Get the information about the network and it's reachable nodes.
+
+        The information includes the network address and the geograpical location
+        of the nodes including the continent/country/city/latitude/longitude.
+
+        API:
+            GET /network_info
+
+        Returns: information about the network and about it's nodes
+
+
+    ***************************************************************************/
+
+    public CrawlResultHolder getNetworkInfo ();
+
     /***************************************************************************
 
         Get validators' pre-image information

source/agora/common/Config.d

@@ -228,6 +228,24 @@ public struct NodeConfig
     /// Transaction put into the relay queue will expire, and will be removed
     /// after `relay_tx_cache_exp`.
     public Duration relay_tx_cache_exp;
+
+    /// true, if this node should collect statistics about other
+    /// nodes in the network, including their geographical location and OS
+    public bool collect_network_statistics;
+
+    /// true, if this node should be included in network statistics
+    public bool include_in_network_statistics;
+
+    /// The number of network crawlers that will be instantiated to collect
+    /// statistics about other nodes in the network
+    public ubyte num_of_crawlers;
+
+    /// The number of seconds one crawler should wait after successfully contacted
+    /// a node
+    public Duration crawling_interval;
+
+    /// The path to a file containing IP address -> geographical location mapping
+    public string ipdb_path;
 }
 
 /// Validator config
@@ -524,6 +542,11 @@ private NodeConfig parseNodeConfig (Node* node, in CommandLine cmdln)
     Duration relay_tx_interval = opt!(ulong, "node", "relay_tx_interval_secs")(cmdln, node, 15).seconds;
     const relay_tx_min_fee = Amount(opt!(ulong, "node", "relay_tx_min_fee")(cmdln, node, 0));
     Duration relay_tx_cache_exp = opt!(ulong, "node", "relay_tx_cache_exp_secs")(cmdln, node, 1200).seconds;
+    const collect_network_statistics = opt!(bool, "node", "collect_network_statistics")(cmdln, node, false);
+    const include_in_network_statistics = opt!(bool, "node", "include_in_network_statistics")(cmdln, node, true);
+    const num_of_crawlers = opt!(ubyte, "node", "num_of_crawlers")(cmdln, node, 1);
+    Duration crawling_interval = opt!(ulong, "node", "crawling_interval_secs")(cmdln, node, 3).seconds;
+    const ipdb_path = opt!(string, "node", "ipdb_path")(cmdln, node, "infrastructure/bigfiles/ipdb.mmdb");
 
     NodeConfig result = {
             min_listeners : min_listeners,
@@ -545,6 +568,11 @@ private NodeConfig parseNodeConfig (Node* node, in CommandLine cmdln)
             relay_tx_interval : relay_tx_interval,
             relay_tx_min_fee : relay_tx_min_fee,
             relay_tx_cache_exp : relay_tx_cache_exp,
+            collect_network_statistics : collect_network_statistics,
+            include_in_network_statistics : include_in_network_statistics,
+            num_of_crawlers : num_of_crawlers,
+            crawling_interval : crawling_interval,
+            ipdb_path : ipdb_path,
     };
     return result;
 }