-
Notifications
You must be signed in to change notification settings - Fork 166
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Khalil/5844 access node fallbacks #1395
Merged
Merged
Changes from 13 commits
Commits
Show all changes
21 commits
Select commit
Hold shift + click to select a range
51846f8
update QCVoter and DKG Broker to handle access node fallbacks
kc1116 e325f99
lint fixes
kc1116 b90ca6b
update localnet
kc1116 43da870
update integration tests
kc1116 7e6ed6c
add missing error return when node fails to prepare flow client conn …
kc1116 40c9850
Update flow_client.go
kc1116 de10bd5
Merge branch 'master' of github.com:onflow/flow-go into khalil/5844-a…
kc1116 8be2355
update dkg tests controller factory client arg
kc1116 6303825
update comments and log statement formatting
kc1116 492c35c
Update flow_client.go
kc1116 91b2c54
add fallback to poll/submitResult
jordanschalm 14b2282
add log
jordanschalm db9970d
fix bug creating flow client opts
jordanschalm 2d8b4c2
maintain ordering of ANs
jordanschalm e925198
update var names
kc1116 db73f45
Merge branch 'master' of github.com:onflow/flow-go into khalil/5844-a…
kc1116 d8c0995
update flag help string, improve logging,
kc1116 d88b2ca
Merge branch 'master' into khalil/5844-access-node-fallbacks
jordanschalm bf0de79
Merge branch 'master' into khalil/5844-access-node-fallbacks
kc1116 f8f992a
Update broker.go
kc1116 78a0825
Merge branch 'khalil/5844-access-node-fallbacks' of github.com:onflow…
kc1116 File filter
Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -95,10 +95,9 @@ func main() { | |
|
||
// DKG contract client | ||
machineAccountInfo *bootstrap.NodeMachineAccountInfo | ||
flowClient *client.Client | ||
accessAddress string | ||
secureAccessNodeID string | ||
flowClientOpts []*common.FlowClientOpt | ||
insecureAccessAPI bool | ||
accessNodeIDS []string | ||
|
||
err error | ||
mutableState protocol.MutableState | ||
|
@@ -145,9 +144,8 @@ func main() { | |
flags.UintVar(&requiredApprovalsForSealVerification, "required-verification-seal-approvals", validation.DefaultRequiredApprovalsForSealValidation, "minimum number of approvals that are required to verify a seal") | ||
flags.UintVar(&requiredApprovalsForSealConstruction, "required-construction-seal-approvals", sealing.DefaultRequiredApprovalsForSealConstruction, "minimum number of approvals that are required to construct a seal") | ||
flags.BoolVar(&emergencySealing, "emergency-sealing-active", sealing.DefaultEmergencySealingActive, "(de)activation of emergency sealing") | ||
flags.StringVar(&accessAddress, "access-address", "", "the address of an access node") | ||
flags.StringVar(&secureAccessNodeID, "secure-access-node-id", "", "the node ID of the secure access GRPC server") | ||
flags.BoolVar(&insecureAccessAPI, "insecure-access-api", true, "required if insecure GRPC connection should be used") | ||
flags.StringSliceVar(&accessNodeIDS, "access-node-ids", []string{}, "array of access node ID's sorted in priority order where the first ID in this array will get the first connection attempt and each subsequent ID after serves as a fallback. minimum length 2") | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same comment: it would be great if this could use the constant (provided it introduces no dependency cycles) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. |
||
flags.DurationVar(&dkgControllerConfig.BaseStartDelay, "dkg-controller-base-start-delay", dkgmodule.DefaultBaseStartDelay, "used to define the range for jitter prior to DKG start (eg. 500µs) - the base value is scaled quadratically with the # of DKG participants") | ||
flags.DurationVar(&dkgControllerConfig.BaseHandleFirstBroadcastDelay, "dkg-controller-base-handle-first-broadcast-delay", dkgmodule.DefaultBaseHandleFirstBroadcastDelay, "used to define the range for jitter prior to DKG handling the first broadcast messages (eg. 50ms) - the base value is scaled quadratically with the # of DKG participants") | ||
flags.DurationVar(&dkgControllerConfig.HandleSubsequentBroadcastDelay, "dkg-controller-handle-subsequent-broadcast-delay", dkgmodule.DefaultHandleSubsequentBroadcastDelay, "used to define the constant delay introduced prior to DKG handling subsequent broadcast messages (eg. 2s)") | ||
|
@@ -367,38 +365,25 @@ func main() { | |
machineAccountInfo, err = cmd.LoadNodeMachineAccountInfoFile(node.BootstrapDir, node.NodeID) | ||
return err | ||
}). | ||
Module("sdk client", func(builder cmd.NodeBuilder, node *cmd.NodeConfig) error { | ||
if accessAddress == "" { | ||
return fmt.Errorf("missing required flag --access-address") | ||
Module("sdk client connection options", func(builder cmd.NodeBuilder, node *cmd.NodeConfig) error { | ||
if len(accessNodeIDS) < common.DefaultAccessNodeIDSMinimum { | ||
return fmt.Errorf("invalid flag --access-node-ids atleast %d IDs must be provided", common.DefaultAccessNodeIDSMinimum) | ||
} | ||
// create flow client with correct GRPC configuration for QC contract client | ||
if insecureAccessAPI { | ||
flowClient, err = common.InsecureFlowClient(accessAddress) | ||
return err | ||
} else { | ||
if secureAccessNodeID == "" { | ||
return fmt.Errorf("invalid flag --secure-access-node-id required") | ||
} | ||
|
||
nodeID, err := flow.HexStringToIdentifier(secureAccessNodeID) | ||
if err != nil { | ||
return fmt.Errorf("could not get flow identifer from secured access node id: %s", secureAccessNodeID) | ||
} | ||
|
||
identities, err := node.State.Sealed().Identities(filter.HasNodeID(nodeID)) | ||
if err != nil { | ||
return fmt.Errorf("could not get identity of secure access node: %s", secureAccessNodeID) | ||
} | ||
|
||
if len(identities) < 1 { | ||
return fmt.Errorf("could not find identity of secure access node: %s", secureAccessNodeID) | ||
} | ||
|
||
flowClient, err = common.SecureFlowClient(accessAddress, identities[0].NetworkPubKey.String()[2:]) | ||
return err | ||
flowClientOpts, err = common.PrepareFlowClientOpts(accessNodeIDS, insecureAccessAPI, node.State.Sealed()) | ||
if err != nil { | ||
return fmt.Errorf("failed to prepare flow client connection options for each access node id %w", err) | ||
} | ||
|
||
return nil | ||
}). | ||
Component("machine account config validator", func(builder cmd.NodeBuilder, node *cmd.NodeConfig) (module.ReadyDoneAware, error) { | ||
//@TODO use fallback logic for flowClient similar to DKG/QC contract clients | ||
flowClient, err := common.FlowClient(flowClientOpts[0]) | ||
if err != nil { | ||
return nil, fmt.Errorf("failed to get flow client connection option for access node (0): %s %w", flowClientOpts[0].AccessAddress, err) | ||
} | ||
|
||
validator, err := epochs.NewMachineAccountConfigValidator( | ||
node.Logger, | ||
flowClient, | ||
|
@@ -744,7 +729,7 @@ func main() { | |
node.ProtocolEvents.AddConsumer(viewsObserver) | ||
|
||
// construct DKG contract client | ||
dkgContractClient, err := createDKGContractClient(node, machineAccountInfo, flowClient) | ||
dkgContractClients, err := createDKGContractClients(node, machineAccountInfo, flowClientOpts) | ||
if err != nil { | ||
return nil, fmt.Errorf("could not create dkg contract client %w", err) | ||
} | ||
|
@@ -759,7 +744,7 @@ func main() { | |
dkgmodule.NewControllerFactory( | ||
node.Logger, | ||
node.Me, | ||
dkgContractClient, | ||
dkgContractClients, | ||
dkgBrokerTunnel, | ||
dkgControllerConfig, | ||
), | ||
|
@@ -789,9 +774,8 @@ func loadDKGPrivateData(dir string, myID flow.Identifier) (*dkg.DKGParticipantPr | |
return &priv, nil | ||
} | ||
|
||
// createDKGContractClient creates a DKG contract client | ||
// createDKGContractClient creates an dkgContractClient | ||
func createDKGContractClient(node *cmd.NodeConfig, machineAccountInfo *bootstrap.NodeMachineAccountInfo, flowClient *client.Client) (module.DKGContractClient, error) { | ||
|
||
var dkgClient module.DKGContractClient | ||
|
||
contracts, err := systemcontracts.SystemContractsForChain(node.RootChainID) | ||
|
@@ -819,3 +803,25 @@ func createDKGContractClient(node *cmd.NodeConfig, machineAccountInfo *bootstrap | |
|
||
return dkgClient, nil | ||
} | ||
|
||
// createDKGContractClients creates an array dkgContractClient that is sorted by retry fallback priority | ||
func createDKGContractClients(node *cmd.NodeConfig, machineAccountInfo *bootstrap.NodeMachineAccountInfo, flowClientOpts []*common.FlowClientOpt) ([]module.DKGContractClient, error) { | ||
dkgClients := make([]module.DKGContractClient, 0) | ||
|
||
for _, opt := range flowClientOpts { | ||
flowClient, err := common.FlowClient(opt) | ||
if err != nil { | ||
return nil, fmt.Errorf("failed to create flow client for dkg contract client with options: %s %w", flowClientOpts, err) | ||
} | ||
|
||
node.Logger.Info().Msgf("created dkg contract client with opts: %s", opt.String()) | ||
dkgClient, err := createDKGContractClient(node, machineAccountInfo, flowClient) | ||
if err != nil { | ||
return nil, fmt.Errorf("failed to create dkg contract client with flow client options: %s %w", flowClientOpts, err) | ||
} | ||
|
||
dkgClients = append(dkgClients, dkgClient) | ||
} | ||
|
||
return dkgClients, nil | ||
} |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Could this refer to
DefaultAccessNodeIDSMinimum
, if there's not dependency cycle?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
d8c0995