Gossipsub testnet (#5)

* add gossipsub

* what i got so far

* requested changes

* took out random task

* fix files

* some edits for testing

* latest but not valid yaml

* fix some not-working stuff.

* latest

* increment ports

* gossipsub working

* lite file

* removed tee output logs

* got rid of miner stuff

* epoch change

* nodiscover and verbosity 6 for quorum

* working except 2 quorum containers stop during test

* removed unnecessary port-mappings

* documenting, cleaning up samples, corrected a besu documentation issue

* changed besu.md doc

* linked to helper

* quorum-besu

* removed unnecessary volume

* added gossipsub md

* uncomment rest of nodes, fix stdout logging

* added timeout in right place according to lint

* added description for pems tooling

* removed quorum besu from master, creating pr from renes fork instead

* first go at priv key helper

* changed path to volume

* working gossip

* reverted change

* changed duration according to regex

* added docs for priv key gen task

Co-authored-by: Daniel Choi <choidanielw@gmail.com>

gossipsub/config_orchestra.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+#sed -i 's/IPLIST/'$IPS'/g' ./configs/orchestra/config.json
+
+eval 'IPFILE=($(cat IP.txt))'
+
+declare -a IPS
+for ip in "${IPFILE[@]}"; do
+  IPS+=($ip:8080,)
+done
+
+IPADDRS=`echo ${IPS[@]} | tr -d '[:space:]' | jq -R -s -c 'split(",")[:-1]'`
+sed -i 's/IPLIST/'$IPADDRS'/g' ./configs/orchestra/config.json

gossipsub/configs/config.json

@@ -0,0 +1 @@
+{"orchestra":{"omitSubnet":true,"hostRPCAddressesIfOmitSubnet":IPLIST,"messageNanoSecondInterval":5000000,"clientTimeoutSeconds":5,"messageLocation":"client.message.json","messageByteSize":1000,"testDurationSeconds":30,"testWarmupSeconds":5,"testCooldownSeconds":30},"subnet":{"numHosts":10,"pubsubCIDR":"127.0.0.1/8","pubsubPortRange":[3000,4000],"rpcCIDR":"127.0.0.1/8","rpcPortRange":[8080,9080],"peerTopology":"whiteblocks"},"host":{"keyType":"ecdsa","rsaBits":2048,"transports":["tcp","ws"],"muxers":[["yamux","/yamux/1.0.0"],["mplex","/mplex/6.7.0"]],"security":"secio","pubsubAlgorithm":"gossip","omitRelay":false,"omitConnectionManager":false,"omitNATPortMap":false,"omitRPCServer":false,"omitDiscoveryService":false,"omitRouting":false},"general":{"loggerLocation":"","debug":true}}

gossipsub/getIP.sh

@@ -0,0 +1,8 @@
+#!/bin/bash -xe
+
+env | grep "GOSSIP_NODE[0-9]_SERVICE0_GOSSIP_NETWORK" > /topology/IP.txt
+#sed -i '' 's/GOSSIP_NODE[0-9]_SERVICE0_GOSSIP_NETWORK=//g' IP.txt
+sed -i 's/GOSSIP_NODE[0-9]_SERVICE0_GOSSIP_NETWORK=//g' /topology/IP.txt
+
+count=`cat /topology/IP.txt | wc -l | sed -e 's/^[[:space:]]*//'`
+echo $((count - 1)) > /topology/NODECOUNT.txt

gossipsub/gossip.md

@@ -0,0 +1,33 @@
+# Gossipsub Testnet
+
+This example shows how to deploy 4 Gossipsub nodes and 1 Orchestra node on a test network.
+
+### Private Key Generation
+The following tool was used to generate the keys used in this test: https://github.com/agencyenterprise/pem-utils. This tool provides utility to create pem files and IPFS id's from them. The private keys in the `pems` directory are pre-generated keys using this tooling.
+
+## Services
+
+### gossipsub
+Libp2p's Gossipsub
+
+### orchestra
+Load injector service
+
+## Task-Runners
+
+### priv-key-gen
+Generates private keys for gossipsub nodes
+
+### init
+Gives Gossipsub nodes 15 seconds to start up
+
+### topology-generator
+Creates a pseudo-randomly generated list of peers for gossipnodes to connect to and generate a deterministic network topology
+
+### run 
+Runs the testnetwork for 12.5 minutes (750 s)
+
+## Tests
+
+### gossip-load-test
+Runs a gossipsub network and sends messages for 12.5 minutes (750 s)

gossipsub/gossip.yaml

@@ -0,0 +1,187 @@
+services:
+  - name: gossip-node0
+    image: "gcr.io/whiteblock/gossipsub:master"
+    environment:
+      SELF: 0
+    volumes:
+      - name: privKeys
+        path: /pk-generator/pems/
+        scope: singleton
+      - name: topologies
+        path: /topology/
+        scope: singleton
+    resources:
+      cpus: 1
+      memory: 4 GB
+      storage: 20 GiB
+    input-files:
+      - source-path: MADDR.txt
+        destination-path: /go-libp2p-pubsub-benchmark-tools/MADDR.txt
+      - source-path: start.sh
+        destination-path: /go-libp2p-pubsub-benchmark-tools/start.sh
+    script:
+      inline: cp /pk-generator/pems/pk-1.pem /go-libp2p-pubsub-benchmark-tools/pk.pem && cp /topology/peers_0.txt /go-libp2p-pubsub-benchmark-tools/peers.txt && cp /topology/NODECOUNT.txt /go-libp2p-pubsub-benchmark-tools/NODECOUNT.txt && cp /topology/IP.txt /go-libp2p-pubsub-benchmark-tools/IP.txt && sudo chmod 777 start.sh && ./start.sh
+  - name: gossip-node1
+    image: "gcr.io/whiteblock/gossipsub:master"
+    environment:
+      SELF: 1
+    volumes:
+      - name: privKeys
+        path: /pk-generator/pems/
+        scope: singleton
+      - name: topologies
+        path: /topology/
+        scope: singleton
+    resources:
+      cpus: 1
+      memory: 4 GB
+      storage: 20 GiB
+    input-files:
+      - source-path: MADDR.txt
+        destination-path: /go-libp2p-pubsub-benchmark-tools/MADDR.txt
+      - source-path: start.sh
+        destination-path: /go-libp2p-pubsub-benchmark-tools/start.sh
+    script:
+      inline: cp /pk-generator/pems/pk-2.pem /go-libp2p-pubsub-benchmark-tools/pk.pem && cp /topology/peers_1.txt /go-libp2p-pubsub-benchmark-tools/peers.txt && cp /topology/NODECOUNT.txt /go-libp2p-pubsub-benchmark-tools/NODECOUNT.txt && cp /topology/IP.txt /go-libp2p-pubsub-benchmark-tools/IP.txt && sudo chmod 777 start.sh && ./start.sh
+  - name: gossip-node2
+    image: "gcr.io/whiteblock/gossipsub:master"
+    environment:
+      SELF: 2
+    volumes:
+      - name: privKeys
+        path: /pk-generator/pems/
+        scope: singleton
+      - name: topologies
+        path: /topology/
+        scope: singleton
+    resources:
+      cpus: 1
+      memory: 4 GB
+      storage: 20 GiB
+    input-files:
+      - source-path: MADDR.txt
+        destination-path: /go-libp2p-pubsub-benchmark-tools/MADDR.txt
+      - source-path: start.sh
+        destination-path: /go-libp2p-pubsub-benchmark-tools/start.sh
+    script:
+      inline: cp /pk-generator/pems/pk-3.pem /go-libp2p-pubsub-benchmark-tools/pk.pem && cp /topology/peers_2.txt /go-libp2p-pubsub-benchmark-tools/peers.txt && cp /topology/NODECOUNT.txt /go-libp2p-pubsub-benchmark-tools/NODECOUNT.txt && cp /topology/IP.txt /go-libp2p-pubsub-benchmark-tools/IP.txt && sudo chmod 777 start.sh && ./start.sh && sleep 600
+  - name: gossip-node3
+    image: "gcr.io/whiteblock/gossipsub:master"
+    environment:
+      SELF: 3
+    volumes:
+      - name: privKeys
+        path: /pk-generator/pems/
+        scope: singleton
+      - name: topologies
+        path: /topology/
+        scope: singleton
+    resources:
+      cpus: 1
+      memory: 4 GB
+      storage: 20 GiB
+    input-files:
+      - source-path: MADDR.txt
+        destination-path: /go-libp2p-pubsub-benchmark-tools/MADDR.txt
+      - source-path: start.sh
+        destination-path: /go-libp2p-pubsub-benchmark-tools/start.sh
+    script:
+      inline: cp /pk-generator/pems/pk-4.pem /go-libp2p-pubsub-benchmark-tools/pk.pem && cp /topology/peers_3.txt /go-libp2p-pubsub-benchmark-tools/peers.txt && cp /topology/NODECOUNT.txt /go-libp2p-pubsub-benchmark-tools/NODECOUNT.txt && cp /topology/IP.txt /go-libp2p-pubsub-benchmark-tools/IP.txt && sudo chmod 777 start.sh && ./start.sh && sleep 600
+  - name: orchestra-node
+    image: "gcr.io/whiteblock/gossipsub:master"
+    volumes:
+      - name: topologies
+        path: /topology/
+        scope: singleton
+    resources:
+      cpus: 1
+      memory: 4 GB
+      storage: 20 GiB
+    input-files:
+      - source-path: orchestra.sh
+        destination-path: /go-libp2p-pubsub-benchmark-tools/orchestra.sh
+      - source-path: ./configs/config.json
+        destination-path: /go-libp2p-pubsub-benchmark-tools/configs/orchestra/config.json
+      - source-path: config_orchestra.sh
+        destination-path: /go-libp2p-pubsub-benchmark-tools/config_orchestra.sh
+    script:
+     inline: sleep 15 && cp /topology/NODECOUNT.txt /go-libp2p-pubsub-benchmark-tools/NODECOUNT.txt && cp /topology/IP.txt /go-libp2p-pubsub-benchmark-tools/IP.txt && sudo chmod 777 ./config_orchestra.sh && ./config_orchestra.sh && sudo chmod 777 ./orchestra.sh && ./orchestra.sh && sleep 120
+task-runners:
+  - name: priv-key-gen
+    image: "gcr.io/whiteblock/helpers/pk-generator/keygen:master"
+    volumes:
+      - name: privKeys
+        path: /pk-generator/pems/
+        scope: singleton
+    script:
+      inline: ./pk-generator generate --numKeys 4
+  - name: topology-generator
+    # image has to include and build the packages that will be necessary to run the helper
+    image: "gcr.io/whiteblock/helpers/topologygen:master"
+    volumes: # stores files peers_*.txt
+      - name: topologies
+        path: /topology/
+        scope: singleton
+    input-files:
+      - source-path: getIP.sh
+        destination-path: /topologyGen/getIP.sh
+    resources:
+      cpus: 1
+      memory: 2 GB
+      storage: 512 MB
+    script: # need to edit this to output the correct number of files for every node in the network
+      inline: python topo.py 42 4 2 true && cp topology/* /topology && bash getIP.sh && sleep 15
+  - name: init
+    script:
+      inline: sleep 15
+  - name: run
+    script:
+      inline: sleep 750
+tests:
+  - name: gossip-load-test
+    description: run a gossip network and send messages
+    timeout: infinite
+    phases:
+      - name: generate-topology-and-keys # generate topologies
+        tasks:
+          - type: topology-generator
+          - type: priv-key-gen
+      - name: gossiping # spin up gossip nodes
+        system:
+          - name: gossip-node0
+            type: gossip-node0
+            count: 1
+            resources:
+              networks:
+              - name: gossip-network
+          - name: gossip-node1
+            type: gossip-node1
+            count: 1
+            resources: 
+              networks:
+              - name: gossip-network
+          - name: gossip-node2
+            type: gossip-node2
+            count: 1
+            resources: 
+              networks:
+              - name: gossip-network
+          - name: gossip-node3
+            type: gossip-node3
+            count: 1
+            resources: 
+              networks:
+              - name: gossip-network
+        tasks:
+          - type: init
+      - name: start-orchestra # add in orchestra node
+        system:
+          - name: orchestra-node
+            type: orchestra-node
+            count: 1
+            resources: 
+              networks:
+              - name: gossip-network
+        tasks:
+         -  type: run
+            timeout: infinite

gossipsub/orchestra.sh

@@ -0,0 +1,34 @@
+#!/bin/bash -xe
+
+RETRY_DELAY=5
+
+retry_run() {
+        set -e
+        while :
+        do
+                $@ && break
+                n=$[$n+1]
+                sleep $RETRY_DELAY
+        done
+        set +e
+}
+
+IFS=$'\r\n' GLOBIGNORE='*' command eval  'IP=($(cat ./IP.txt))'
+echo ${IP[@]:0:EXP}
+IFS=$'\r\n' GLOBIGNORE='*' command eval  'COUNT=($(cat ./NODECOUNT.txt))'
+echo ${COUNT[@]}
+
+check() {
+  for i in $(seq 0 $COUNT)
+  do
+    retry_run ./cmd/client/client -p ${IP[i]}:8080 id
+  done
+}
+
+send() {
+  ./cmd/orchestra/orchestra start
+# --log /topology/orchestra_output.log
+}
+
+check
+send

gossipsub/start.sh

@@ -0,0 +1,41 @@
+#!/bin/bash -xe
+
+retry_run() {
+        n=0
+        set -e
+        until [ $n -ge $RETRIES ]
+        do
+          $@ && break
+          n=$[$n+1]
+          sleep $RETRY_DELAY
+        done
+        set +e
+}
+
+IFS=$'\r\n' GLOBIGNORE='*' command eval  'IP=($(cat ./IP.txt))'
+# echo ${IP[@]:0:COUNT}
+IFS=$'\r\n' GLOBIGNORE='*' command eval  'MADDR=($(cat ./MADDR.txt))'
+# echo ${MADDR[@]:0:COUNT}
+IFS=$'\r\n' GLOBIGNORE='*' command eval  'PEERS=($(cat ./peers.txt))'
+# echo ${PEERS[@]:0:COUNT}
+
+deploy_host() {
+  ./cmd/host/host --pem ./pk.pem 
+#--log /topology/output_$SELF.log
+}
+
+peer() {
+  for peer in ${PEERS[@]}
+  do if [ $peer != $SELF ]
+    then tmux new -s host -d; tmux send-keys -thost "./cmd/client/client open-peers /ip4/${IP[peer]}/tcp/3000/ipfs/${MADDR[peer]}" C-m
+  fi
+  done
+}
+
+start() {
+  deploy_host
+  sleep 10
+  peer
+}
+
+start