Permalink
Browse files

[p2p/pex] connect to more than 10 peers (#2169)

* [p2p/pex] connect to more than 10 peers

also, remove DefaultMinNumOutboundPeers because a) I am not sure it's
needed b) it's super confusing

look closely

```
maxPeers := sw.config.MaxNumPeers - DefaultMinNumOutboundPeers
if maxPeers <= sw.peers.Size() {
  sw.Logger.Info("Ignoring inbound connection: already have enough peers", "address", inConn.RemoteAddr().String(), "numPeers", sw.peers.Size(), "max", maxPeers)
```

we print maxPeers = config.MaxPeers - DefaultMinNumOutboundPeers. So we
may not have enough peers even though we say we have enough.

Refs #2130

* update spec

* replace MaxNumPeers with MaxNumInboundPeers/MaxNumOutboundPeers

Refs #2130

* update changelog

* make max rpc conns formula visible to users

* update spec

* docs: note max outbound peers excludes persistent
  • Loading branch information...
melekes authored and ebuchman committed Aug 14, 2018
1 parent db53dc5 commit 6fad8eaf5a7d82000c3f2933ec61e0f3917d07cf
Showing with 60 additions and 32 deletions.
  1. +1 −0 CHANGELOG_PENDING.md
  2. +11 −7 config/config.go
  3. +9 −2 config/toml.go
  4. +13 −8 docs/spec/reactors/pex/pex.md
  5. +10 −3 docs/tendermint-core/configuration.md
  6. +2 −3 p2p/pex/pex_reactor.go
  7. +14 −9 p2p/switch.go
@@ -10,6 +10,7 @@ BREAKING CHANGES:
- [abci] Added address of the original proposer of the block to Header.
- [abci] Change ABCI Header to match Tendermint exactly
- [libs] Remove cmn.Fmt, in favor of fmt.Sprintf
- [config] Replace MaxNumPeers with MaxNumInboundPeers and MaxNumOutboundPeers

FEATURES:

@@ -239,6 +239,8 @@ type RPCConfig struct {
// If you want to accept more significant number than the default, make sure
// you increase your OS limits.
// 0 - unlimited.
// Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files}
// 1024 - 40 - 10 - 50 = 924 = ~900
MaxOpenConnections int `mapstructure:"max_open_connections"`
}

@@ -248,11 +250,9 @@ func DefaultRPCConfig() *RPCConfig {
ListenAddress: "tcp://0.0.0.0:26657",

GRPCListenAddress: "",
GRPCMaxOpenConnections: 900, // no ipv4
GRPCMaxOpenConnections: 900,

Unsafe: false,
// should be < {ulimit -Sn} - {MaxNumPeers} - {N of wal, db and other open files}
// 1024 - 50 - 50 = 924 = ~900
Unsafe: false,
MaxOpenConnections: 900,
}
}
@@ -295,8 +295,11 @@ type P2PConfig struct {
// Set true for strict address routability rules
AddrBookStrict bool `mapstructure:"addr_book_strict"`

// Maximum number of peers to connect to
MaxNumPeers int `mapstructure:"max_num_peers"`
// Maximum number of inbound peers
MaxNumInboundPeers int `mapstructure:"max_num_inbound_peers"`

// Maximum number of outbound peers to connect to, excluding persistent peers
MaxNumOutboundPeers int `mapstructure:"max_num_outbound_peers"`

// Time to wait before flushing messages out on the connection, in ms
FlushThrottleTimeout int `mapstructure:"flush_throttle_timeout"`
@@ -346,7 +349,8 @@ func DefaultP2PConfig() *P2PConfig {
UPNP: false,
AddrBook: defaultAddrBookPath,
AddrBookStrict: true,
MaxNumPeers: 50,
MaxNumInboundPeers: 40,
MaxNumOutboundPeers: 10,
FlushThrottleTimeout: 100,
MaxPacketMsgPayloadSize: 1024, // 1 kB
SendRate: 5120000, // 5 mB/s
@@ -124,6 +124,8 @@ grpc_laddr = "{{ .RPC.GRPCListenAddress }}"
# If you want to accept more significant number than the default, make sure
# you increase your OS limits.
# 0 - unlimited.
# Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files}
# 1024 - 40 - 10 - 50 = 924 = ~900
grpc_max_open_connections = {{ .RPC.GRPCMaxOpenConnections }}
# Activate unsafe RPC commands like /dial_seeds and /unsafe_flush_mempool
@@ -134,6 +136,8 @@ unsafe = {{ .RPC.Unsafe }}
# If you want to accept more significant number than the default, make sure
# you increase your OS limits.
# 0 - unlimited.
# Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files}
# 1024 - 40 - 10 - 50 = 924 = ~900
max_open_connections = {{ .RPC.MaxOpenConnections }}
##### peer to peer configuration options #####
@@ -166,8 +170,11 @@ addr_book_strict = {{ .P2P.AddrBookStrict }}
# Time to wait before flushing messages out on the connection, in ms
flush_throttle_timeout = {{ .P2P.FlushThrottleTimeout }}
# Maximum number of peers to connect to
max_num_peers = {{ .P2P.MaxNumPeers }}
# Maximum number of inbound peers
max_num_inbound_peers = {{ .P2P.MaxNumInboundPeers }}
# Maximum number of outbound peers to connect to, excluding persistent peers
max_num_outbound_peers = {{ .P2P.MaxNumOutboundPeers }}
# Maximum size of a message packet payload, in bytes
max_packet_msg_payload_size = {{ .P2P.MaxPacketMsgPayloadSize }}
@@ -15,6 +15,9 @@ we will not put them in the address book or gossip them to others.
All peers except private peers and peers coming from them are tracked using the
address book.

The rest of our peers are only distinguished by being either
inbound (they dialed our public address) or outbound (we dialed them).

## Discovery

Peer discovery begins with a list of seeds.
@@ -26,15 +29,15 @@ and will attempt to maintain persistent connections with them. If the connection
we will redial every 5s for a few minutes, then switch to an exponential backoff schedule,
and after about a day of trying, stop dialing the peer.

So long as we have less than `MinNumOutboundPeers`, we periodically request additional peers
So long as we have less than `MaxNumOutboundPeers`, we periodically request additional peers
from each of our own. If sufficient time goes by and we still can't find enough peers,
we try the seeds again.

## Listening

Peers listen on a configurable ListenAddr that they self-report in their
NodeInfo during handshakes with other peers. Peers accept up to (MaxNumPeers -
MinNumOutboundPeers) incoming peers.
NodeInfo during handshakes with other peers. Peers accept up to
`MaxNumInboundPeers` incoming peers.

## Address Book

@@ -73,10 +76,11 @@ a trust metric (see below), but it's best to start with something simple.

## Select Peers to Dial

When we need more peers, we pick them randomly from the addrbook with some
configurable bias for unvetted peers. The bias should be lower when we have fewer peers
and can increase as we obtain more, ensuring that our first peers are more trustworthy,
but always giving us the chance to discover new good peers.
When we need more peers, we pick addresses randomly from the addrbook with some
configurable bias for unvetted peers. The bias should be lower when we have
fewer peers and can increase as we obtain more, ensuring that our first peers
are more trustworthy, but always giving us the chance to discover new good
peers.

We track the last time we dialed a peer and the number of unsuccessful attempts
we've made. If too many attempts are made, we mark the peer as bad.
@@ -85,7 +89,8 @@ Connection attempts are made with exponential backoff (plus jitter). Because
the selection process happens every `ensurePeersPeriod`, we might not end up
dialing a peer for much longer than the backoff duration.

If we fail to connect to the peer after 16 tries (with exponential backoff), we remove from address book completely.
If we fail to connect to the peer after 16 tries (with exponential backoff), we
remove from address book completely.

## Select Peers to Exchange

@@ -77,6 +77,8 @@ grpc_laddr = ""
# If you want to accept more significant number than the default, make sure
# you increase your OS limits.
# 0 - unlimited.
# Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files}
# 1024 - 40 - 10 - 50 = 924 = ~900
grpc_max_open_connections = 900
# Activate unsafe RPC commands like /dial_seeds and /unsafe_flush_mempool
@@ -87,7 +89,9 @@ unsafe = false
# If you want to accept more significant number than the default, make sure
# you increase your OS limits.
# 0 - unlimited.
max_open_connections = 450
# Should be < {ulimit -Sn} - {MaxNumInboundPeers} - {MaxNumOutboundPeers} - {N of wal, db and other open files}
# 1024 - 40 - 10 - 50 = 924 = ~900
max_open_connections = 900
##### peer to peer configuration options #####
[p2p]
@@ -113,8 +117,11 @@ addr_book_strict = true
# Time to wait before flushing messages out on the connection, in ms
flush_throttle_timeout = 100
# Maximum number of peers to connect to
max_num_peers = 50
# Maximum number of inbound peers
max_num_inbound_peers = 40
# Maximum number of outbound peers to connect to
max_num_outbound_peers = 10
# Maximum size of a message packet payload, in bytes
max_packet_msg_payload_size = 1024
@@ -31,8 +31,7 @@ const (
maxMsgSize = maxAddressSize * maxGetSelection

// ensure we have enough peers
defaultEnsurePeersPeriod = 30 * time.Second
defaultMinNumOutboundPeers = p2p.DefaultMinNumOutboundPeers
defaultEnsurePeersPeriod = 30 * time.Second

// Seed/Crawler constants

@@ -362,7 +361,7 @@ func (r *PEXReactor) ensurePeersRoutine() {
func (r *PEXReactor) ensurePeers() {
var (
out, in, dial = r.Switch.NumPeers()
numToDial = defaultMinNumOutboundPeers - (out + dial)
numToDial = r.Switch.MaxNumOutboundPeers() - (out + dial)
)
r.Logger.Info(
"Ensure peers",
@@ -26,10 +26,6 @@ const (
// ie. 3**10 = 16hrs
reconnectBackOffAttempts = 10
reconnectBackOffBaseSeconds = 3

// keep at least this many outbound peers
// TODO: move to config
DefaultMinNumOutboundPeers = 10
)

//-----------------------------------------------------------------------------
@@ -268,6 +264,11 @@ func (sw *Switch) NumPeers() (outbound, inbound, dialing int) {
return
}

// MaxNumOutboundPeers returns a maximum number of outbound peers.
func (sw *Switch) MaxNumOutboundPeers() int {
return sw.config.MaxNumOutboundPeers
}

// Peers returns the set of peers that are connected to the switch.
func (sw *Switch) Peers() IPeerSet {
return sw.peers
@@ -491,11 +492,15 @@ func (sw *Switch) listenerRoutine(l Listener) {
break
}

// ignore connection if we already have enough
// leave room for MinNumOutboundPeers
maxPeers := sw.config.MaxNumPeers - DefaultMinNumOutboundPeers
if maxPeers <= sw.peers.Size() {
sw.Logger.Info("Ignoring inbound connection: already have enough peers", "address", inConn.RemoteAddr().String(), "numPeers", sw.peers.Size(), "max", maxPeers)
// Ignore connection if we already have enough peers.
_, in, _ := sw.NumPeers()
if in >= sw.config.MaxNumInboundPeers {
sw.Logger.Info(
"Ignoring inbound connection: already have enough inbound peers",
"address", inConn.RemoteAddr().String(),
"have", in,
"max", sw.config.MaxNumInboundPeers,
)
inConn.Close()
continue
}

0 comments on commit 6fad8ea

Please sign in to comment.