add explainer for direct connections

p2p/README.md

@@ -0,0 +1,91 @@
+P2P
+===
+
+### Direct connections
+
+If you are running multiple nodes in the local network and exposing all of them
+is not convenient there is an option to setup network manually, by making
+couple of nodes publicly available and the rest connected to them directly.
+
+### Get network id
+- get it with grpcurl
+
+> grpcurl -plaintext 127.0.0.1:9092 spacemesh.v1.DebugService.NetworkInfo
+
+```json
+{
+  "id": "12D3KooWRfy4Sj4rDHDuBaYw3Mg5d2puwiCyqBCWMziFquaGQ5g8"
+}
+```
+
+- get it from stored data
+
+> cat ~/spacemesh/p2p/p2p.key
+```json
+{"Key":"CAESQAQN38GXvr+L+G+/JWoimpqPBK7I6INe+PKYA+hRJg0I65Q3IPK49Ii9dcnC3+UqB+jMEL16sqDfUubxTs62rZU=","ID":"12D3KooWRfy4Sj4rDHDuBaYw3Mg5d2puwiCyqBCWMziFquaGQ5g8"}
+```
+
+### Configuration for public node
+
+Public node should have higher peer limits to help with network connectivity
+and a list of botnodes. Direct connections should be reciprocal, otherwise
+public node may prune your private node if overloaded. 
+
+Setup more than one public node, and perform rolling upgrades or restart
+for them if needed.
+
+```json
+{
+    "p2p": {
+        "min-peers": 30,
+        "low-peers": 60,
+        "high-peers": 100,
+        "p2p-disable-legacy-discovery": true,
+        "direct": [
+            "/ip4/0.0.0.0/tcp/6000/p2p/12D3KooWRkBh6QayKLb1pDRJGMHE94Lix4ZBVh2BJJeX6mghk8VH"
+        ],
+        "bootnodes": [
+            "/dns4/mainnet-bootnode-10.spacemesh.network/tcp/5000/p2p/12D3KooWHK5m83sNj2eNMJMGAngcS9gBja27ho83t79Q2CD4iRjQ",
+            "/dns4/mainnet-bootnode-11.spacemesh.network/tcp/5000/p2p/12D3KooWFrCDS8tc29nxJEYf4sKFXhXw7wMSdhQP4S7tsbfh6ngn"
+        ]
+    }
+}
+```
+
+### Configuration for private node
+
+Set min-peers to the number of peers in the config and disable-dht.
+low-peers and high-peers should not be lower than min-peers.
+
+```json
+{
+    "p2p": {
+        "listen": "/ip4/0.0.0.0/tcp/6000",
+        "min-peers": 1,
+        "low-peers": 10,
+        "high-peers": 20,
+        "p2p-disable-legacy-discovery": true,
+        "disable-dht": true,
+        "bootnodes": [],
+        "direct": [
+            "/ip4/0.0.0.0/tcp/7513/p2p/12D3KooWRfy4Sj4rDHDuBaYw3Mg5d2puwiCyqBCWMziFquaGQ5g8"
+        ]
+    }
+}
+```
+
+#### Expected result
+
+Public node will maintain many open connections
+
+> ss -npO4 | rg spacemesh | rg 7513 | rg ESTAB | wc -l
+> 52
+
+Private will connect only to the specified public node:
+
+> ss -npO4 | rg spacemesh | rg 6000
+
+```
+tcp   ESTAB      0      0              127.0.0.1:7513        127.0.0.1:6000  users:(("go-spacemesh",pid=39165,fd=11)) 
+tcp   ESTAB      0      0              127.0.0.1:6000        127.0.0.1:7513  users:(("go-spacemesh",pid=39202,fd=47))
+```

p2p/dhtdiscovery/discovery.go

@@ -90,6 +90,12 @@ func WithDir(path string) Opt {
 	}
 }
 
+func DisableDHT() Opt {
+	return func(d *Discovery) {
+		d.disableDht = true
+	}
+}
+
 func New(h host.Host, opts ...Opt) (*Discovery, error) {
 	ctx, cancel := context.WithCancel(context.Background())
 	d := Discovery{
@@ -110,18 +116,21 @@ func New(h host.Host, opts ...Opt) (*Discovery, error) {
 	if len(d.bootnodes) == 0 {
 		d.logger.Warn("no bootnodes in the config")
 	}
-	dht, err := newDht(ctx, h, d.public, d.server, d.dir)
-	if err != nil {
-		return nil, err
+	if !d.disableDht {
+		dht, err := newDht(ctx, h, d.public, d.server, d.dir)
+		if err != nil {
+			return nil, err
+		}
+		d.dht = dht
 	}
-	d.dht = dht
 	return &d, nil
 }
 
 type Discovery struct {
-	public bool
-	server bool
-	dir    string
+	public     bool
+	server     bool
+	disableDht bool
+	dir        string
 
 	logger *zap.Logger
 	eg     errgroup.Group
@@ -176,6 +185,10 @@ func (d *Discovery) Start() {
 				)
 				if connected >= d.highPeers {
 					for _, boot := range d.bootnodes {
+						// preserve connection is bootnode is in direct peers
+						if _, exist := direct[boot.ID]; exist {
+							continue
+						}
 						if err := d.h.Network().ClosePeer(boot.ID); err != nil {
 							d.logger.Warn("failed to close bootnode connection",
 								zap.Stringer("address", boot), zap.Error(err))
@@ -198,12 +211,17 @@ func (d *Discovery) Start() {
 func (d *Discovery) Stop() {
 	d.cancel()
 	d.eg.Wait()
-	if err := d.dht.Close(); err != nil {
-		d.logger.Error("error closing dht", zap.Error(err))
+	if d.dht != nil {
+		if err := d.dht.Close(); err != nil {
+			d.logger.Error("error closing dht", zap.Error(err))
+		}
 	}
 }
 
 func (d *Discovery) bootstrap() {
+	if d.dht == nil {
+		return
+	}
 	ctx, cancel := context.WithTimeout(d.ctx, d.bootstrapDuration)
 	defer cancel()
 	if err := d.dht.Bootstrap(ctx); err != nil {
@@ -217,6 +235,10 @@ func (d *Discovery) connect(eg *errgroup.Group, nodes []peer.AddrInfo) {
 	defer cancel()
 	for _, boot := range nodes {
 		boot := boot
+		if boot.ID == d.h.ID() {
+			d.logger.Debug("not dialing self")
+			continue
+		}
 		eg.Go(func() error {
 			if err := d.h.Connect(ctx, boot); err != nil {
 				d.logger.Warn("failed to connect",

p2p/host.go

@@ -60,6 +60,7 @@ type Config struct {
 	// see https://lwn.net/Articles/542629/ for reuseport explanation
 	DisableReusePort       bool        `mapstructure:"disable-reuseport"`
 	DisableNatPort         bool        `mapstructure:"disable-natport"`
+	DisableDHT             bool        `mapstructure:"disable-dht"`
 	Flood                  bool        `mapstructure:"flood"`
 	Listen                 string      `mapstructure:"listen"`
 	Bootnodes              []string    `mapstructure:"bootnodes"`

p2p/upgrade.go

@@ -93,7 +93,7 @@ func Upgrade(h host.Host, opts ...Opt) (*Host, error) {
 	if err != nil {
 		return nil, err
 	}
-	direct, err := parseIntoAddr(fh.cfg.Bootnodes)
+	direct, err := parseIntoAddr(fh.cfg.Direct)
 	if err != nil {
 		return nil, err
 	}
@@ -133,6 +133,9 @@ func Upgrade(h host.Host, opts ...Opt) (*Host, error) {
 	if cfg.PrivateNetwork {
 		dopts = append(dopts, discovery.Private())
 	}
+	if cfg.DisableDHT {
+		dopts = append(dopts, discovery.DisableDHT())
+	}
 	if cfg.Bootnode {
 		dopts = append(dopts, discovery.Server())
 	} else {