connecting unstaked AN with the staked AN using DHT for peer discovery #1098
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,17 +2,18 @@ package node_builder | |
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "fmt" | ||
| "strings" | ||
| "time" | ||
|
|
||
| dht "github.com/libp2p/go-libp2p-kad-dht" | ||
| "github.com/onflow/flow/protobuf/go/flow/access" | ||
| "github.com/spf13/pflag" | ||
| "google.golang.org/grpc" | ||
| "google.golang.org/grpc/credentials" | ||
|
|
||
| "github.com/onflow/flow-go/cmd" | ||
| "github.com/onflow/flow-go/consensus" | ||
| "github.com/onflow/flow-go/consensus/hotstuff" | ||
| "github.com/onflow/flow-go/consensus/hotstuff/committees" | ||
|
|
@@ -42,6 +43,7 @@ import ( | |
| "github.com/onflow/flow-go/network" | ||
| jsoncodec "github.com/onflow/flow-go/network/codec/json" | ||
| "github.com/onflow/flow-go/network/p2p" | ||
| "github.com/onflow/flow-go/network/validator" | ||
| "github.com/onflow/flow-go/state/protocol" | ||
| badgerState "github.com/onflow/flow-go/state/protocol/badger" | ||
| "github.com/onflow/flow-go/state/protocol/blocktimer" | ||
|
|
@@ -84,7 +86,9 @@ type AccessNodeBuilder interface { | |
| // while for a node running as a library, the config fields are expected to be initialized by the caller. | ||
| type AccessNodeConfig struct { | ||
| staked bool | ||
| bootstrapNodeAddresses []string | ||
| bootstrapNodePublicKeys []string | ||
| bootstrapIdentites flow.IdentityList // the identity list of bootstrap peers the node uses to discover other nodes | ||
| unstakedNetworkBindAddr string | ||
| collectionGRPCPort uint | ||
| executionGRPCPort uint | ||
|
|
@@ -131,7 +135,8 @@ func DefaultAccessNodeConfig() *AccessNodeConfig { | |
| apiRatelimits: nil, | ||
| apiBurstlimits: nil, | ||
| staked: true, | ||
| bootstrapNodeAddresses: []string{}, | ||
| bootstrapNodePublicKeys: []string{}, | ||
| unstakedNetworkBindAddr: cmd.NotSet, | ||
| } | ||
| } | ||
|
|
@@ -144,6 +149,7 @@ type FlowAccessNodeBuilder struct { | |
| *AccessNodeConfig | ||
|
|
||
| // components | ||
| UnstakedLibP2PNode *p2p.Node | ||
| UnstakedNetwork *p2p.Network | ||
| unstakedMiddleware *p2p.Middleware | ||
| FollowerState protocol.MutableState | ||
|
|
@@ -483,9 +489,9 @@ func (anb *FlowAccessNodeBuilder) Build() AccessNodeBuilder { | |
|
|
||
| type Option func(*AccessNodeConfig) | ||
|
|
||
| func WithBootStrapPeers(bootstrapNodes ...*flow.Identity) Option { | ||
| return func(config *AccessNodeConfig) { | ||
| config.bootstrapIdentites = bootstrapNodes | ||
| } | ||
| } | ||
|
|
||
|
|
@@ -563,71 +569,59 @@ func (builder *FlowAccessNodeBuilder) extraFlags() { | |
| flags.StringToIntVar(&builder.apiRatelimits, "api-rate-limits", defaultConfig.apiRatelimits, "per second rate limits for Access API methods e.g. Ping=300,GetTransaction=500 etc.") | ||
| flags.StringToIntVar(&builder.apiBurstlimits, "api-burst-limits", defaultConfig.apiBurstlimits, "burst limits for Access API methods e.g. Ping=100,GetTransaction=100 etc.") | ||
| flags.BoolVar(&builder.staked, "staked", defaultConfig.staked, "whether this node is a staked access node or not") | ||
| flags.StringSliceVar(&builder.bootstrapNodeAddresses, "bootstrap-node-addresses", defaultConfig.bootstrapNodeAddresses, "the network addresses of the bootstrap access node if this is an unstaked access node e.g. access-001.mainnet.flow.org:9653,access-002.mainnet.flow.org:9653") | ||
| flags.StringSliceVar(&builder.bootstrapNodePublicKeys, "bootstrap-node-public-keys", defaultConfig.bootstrapNodePublicKeys, "the networking public key of the bootstrap access node if this is an unstaked access node (in the same order as the bootstrap node addresses) e.g. \"d57a5e9c5.....\",\"44ded42d....\"") | ||
| flags.StringVar(&builder.unstakedNetworkBindAddr, "unstaked-bind-addr", defaultConfig.unstakedNetworkBindAddr, "address to bind on for the unstaked network") | ||
| }) | ||
| } | ||
|
|
||
| // initLibP2PFactory creates the LibP2P factory function for the given node ID and network key. | ||
| // The factory function is later passed into the initMiddleware function to eventually instantiate the p2p.LibP2PNode instance | ||
| func (builder *FlowAccessNodeBuilder) initLibP2PFactory(ctx context.Context, | ||
| nodeID flow.Identifier, | ||
| networkKey crypto.PrivateKey) (p2p.LibP2PFactoryFunc, error) { | ||
|
|
||
| // The staked nodes act as the DHT servers | ||
| dhtOptions := []dht.Option{p2p.AsServer(builder.IsStaked())} | ||
|
|
||
| // if this is an unstaked access node, then seed the DHT with the boostrap identities | ||
| if !builder.IsStaked() { | ||
| bootstrapPeersOpt, err := p2p.WithBootstrapPeers(builder.bootstrapIdentites) | ||
| builder.MustNot(err) | ||
| dhtOptions = append(dhtOptions, bootstrapPeersOpt) | ||
| } | ||
|
|
||
| return func() (*p2p.Node, error) { | ||
| libp2pNode, err := p2p.NewDefaultLibP2PNodeBuilder(nodeID, builder.unstakedNetworkBindAddr, networkKey). | ||
| SetRootBlockID(builder.RootBlock.ID().String()). | ||
| // unlike the staked network where currently all the node addresses are known upfront, | ||
| // for the unstaked network the nodes need to discover each other using DHT Discovery. | ||
| SetPubsubOptions(p2p.WithDHTDiscovery(dhtOptions...)). | ||
| SetLogger(builder.Logger). | ||
| Build(ctx) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| builder.UnstakedLibP2PNode = libp2pNode | ||
| return builder.UnstakedLibP2PNode, nil | ||
| }, nil | ||
| } | ||
|
|
||
| // initMiddleware creates the network.Middleware implementation with the libp2p factory function, metrics, peer update | ||
| // interval, and validators. The network.Middleware is then passed into the initNetwork function. | ||
| func (builder *FlowAccessNodeBuilder) initMiddleware(nodeID flow.Identifier, | ||
| networkMetrics module.NetworkMetrics, | ||
| factoryFunc p2p.LibP2PFactoryFunc, | ||
| validators ...network.MessageValidator) *p2p.Middleware { | ||
| builder.unstakedMiddleware = p2p.NewMiddleware(builder.Logger, | ||
| factoryFunc, | ||
| nodeID, | ||
| networkMetrics, | ||
| builder.RootBlock.ID().String(), | ||
| time.Hour, // TODO: this is pretty meaningless since there is no peermanager in play. | ||
| p2p.DefaultUnicastTimeout, | ||
| false, // no connection gating for the unstaked network | ||
| false, // no peer management for the unstaked network (peer discovery will be done via LibP2P discovery mechanism) | ||
| validators...) | ||
| return builder.unstakedMiddleware | ||
| } | ||
|
|
@@ -661,3 +655,49 @@ func (builder *FlowAccessNodeBuilder) initNetwork(nodeID module.Local, | |
|
|
||
| return net, nil | ||
| } | ||
|
|
||
| func unstakedNetworkMsgValidators(selfID flow.Identifier) []network.MessageValidator { | ||
| return []network.MessageValidator{ | ||
| // filter out messages sent by this node itself | ||
| validator.ValidateNotSender(selfID), | ||
| } | ||
|
Comment on lines
+660
to
+663
There was a problem hiding this comment. We probably need something better here? We have two scenarios:
The validation logic should be something like: "If the origin ID is from the Identity list in the protocol state, then accept, otherwise only accept if I am one of the targets". This relies on #1115 being done, which I think Francois is working on. This can be done in a later PR though. I'll create an issue for it. |
||
| } | ||
|
|
||
| // BootstrapIdentities converts the bootstrap node addresses and keys to a Flow Identity list where | ||
| // each Flow Identity is initialized with the passed address, the networking key | ||
| // and the Node ID set to ZeroID, role set to Access, 0 stake and no staking key. | ||
| func BootstrapIdentities(addresses []string, keys []string) (flow.IdentityList, error) { | ||
|
|
||
| if len(addresses) != len(keys) { | ||
| return nil, fmt.Errorf("number of addresses and keys provided for the boostrap nodes don't match") | ||
| } | ||
|
|
||
| ids := make([]*flow.Identity, len(addresses)) | ||
| for i, address := range addresses { | ||
|
|
||
| key := keys[i] | ||
| // json unmarshaller needs a quotes before and after the string | ||
| // the pflags.StringSliceVar does not retain quotes for the command line arg even if escaped with \" | ||
| // hence this additional check to ensure the key is indeed quoted | ||
| if !strings.HasPrefix(key, "\"") { | ||
| key = fmt.Sprintf("\"%s\"", key) | ||
| } | ||
| // networking public key | ||
| var networkKey encodable.NetworkPubKey | ||
| err := json.Unmarshal([]byte(key), &networkKey) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| // create the identity of the peer by setting only the relevant fields | ||
| id := &flow.Identity{ | ||
| NodeID: flow.ZeroID, // the NodeID is the hash of the staking key and for the unstaked network it does not apply | ||
| Address: address, | ||
| Role: flow.RoleAccess, // the upstream node has to be an access node | ||
| NetworkPubKey: networkKey, | ||
| } | ||
|
|
||
| ids = append(ids, id) | ||
| } | ||
| return ids, nil | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
now the unstaked node only needs the bootstrap node addresses and the public keys of those nodes.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I don't think that is an improvement at all. Now, we have to manage multiple node identities with public keys on the unstaked consensus follower side. That doesn't make sense. Knowing a single access node address should be the only dependency needed. The DHT information should not be information that you need to connect; it should be served by the access node we connect to.