Khalil/5844 access node fallbacks

cmd/collection/main.go

@@ -87,11 +87,10 @@ func main() {
 		err               error
 
 		// epoch qc contract client
-		accessAddress      string
-		secureAccessNodeID string
-		insecureAccessAPI  bool
 		machineAccountInfo *bootstrap.NodeMachineAccountInfo
-		flowClient         *client.Client
+		flowClientOpts     []*common.FlowClientConfig
+		insecureAccessAPI  bool
+		accessNodeIDS      []string
 	)
 
 	nodeBuilder := cmd.FlowNode(flow.RoleCollection.String())
@@ -146,9 +145,8 @@ func main() {
 		flags.StringVar(&startupTimeString, "hotstuff-startup-time", cmd.NotSet, "specifies date and time (in ISO 8601 format) after which the consensus participant may enter the first view (e.g (e.g 1996-04-24T15:04:05-07:00))")
 
 		// epoch qc contract flags
-		flags.StringVar(&accessAddress, "access-address", "", "the address of an access node")
-		flags.StringVar(&secureAccessNodeID, "secure-access-node-id", "", "the node ID of the secure access GRPC server")
-		flags.BoolVar(&insecureAccessAPI, "insecure-access-api", true, "required if insecure GRPC connection should be used")
+		flags.BoolVar(&insecureAccessAPI, "insecure-access-api", false, "required if insecure GRPC connection should be used")
+		flags.StringSliceVar(&accessNodeIDS, "access-node-ids", []string{}, fmt.Sprintf("array of access node ID's sorted in priority order where the first ID in this array will get the first connection attempt and each subsequent ID after serves as a fallback. minimum length %d", common.DefaultAccessNodeIDSMinimum))
 	}).ValidateFlags(func() error {
 		if startupTimeString != cmd.NotSet {
 			t, err := time.Parse(time.RFC3339, startupTimeString)
@@ -204,44 +202,32 @@ func main() {
 			machineAccountInfo, err = cmd.LoadNodeMachineAccountInfoFile(node.BootstrapDir, node.NodeID)
 			return err
 		}).
-		Module("sdk client", func(builder cmd.NodeBuilder, node *cmd.NodeConfig) error {
-			if accessAddress == "" {
-				return fmt.Errorf("missing required flag --access-address")
+		Module("sdk client connection options", func(builder cmd.NodeBuilder, node *cmd.NodeConfig) error {
+			if len(accessNodeIDS) < common.DefaultAccessNodeIDSMinimum {
+				return fmt.Errorf("invalid flag --access-node-ids atleast %d IDs must be provided", common.DefaultAccessNodeIDSMinimum)
 			}
-			// create flow client with correct GRPC configuration for QC contract client
-			if insecureAccessAPI {
-				flowClient, err = common.InsecureFlowClient(accessAddress)
-				return err
-			} else {
-				if secureAccessNodeID == "" {
-					return fmt.Errorf("invalid flag --secure-access-node-id required")
-				}
-
-				nodeID, err := flow.HexStringToIdentifier(secureAccessNodeID)
-				if err != nil {
-					return fmt.Errorf("could not get flow identifer from secured access node id: %s", secureAccessNodeID)
-				}
-
-				identities, err := node.State.Sealed().Identities(filter.HasNodeID(nodeID))
-				if err != nil {
-					return fmt.Errorf("could not get identity of secure access node: %s", secureAccessNodeID)
-				}
-
-				if len(identities) < 1 {
-					return fmt.Errorf("could not find identity of secure access node: %s", secureAccessNodeID)
-				}
-
-				flowClient, err = common.SecureFlowClient(accessAddress, identities[0].NetworkPubKey.String()[2:])
-				return err
+
+			flowClientOpts, err = common.FlowClientConfigs(accessNodeIDS, insecureAccessAPI, node.State.Sealed())
+			if err != nil {
+				return fmt.Errorf("failed to prepare flow client connection options for each access node id %w", err)
 			}
+
+			return nil
 		}).
 		Component("machine account config validator", func(builder cmd.NodeBuilder, node *cmd.NodeConfig) (module.ReadyDoneAware, error) {
+			//@TODO use fallback logic for flowClient similar to DKG/QC contract clients
+			flowClient, err := common.FlowClient(flowClientOpts[0])
+			if err != nil {
+				return nil, fmt.Errorf("failed to get flow client connection option for access node (0): %s %w", flowClientOpts[0].AccessAddress, err)
+			}
+
 			validator, err := epochs.NewMachineAccountConfigValidator(
 				node.Logger,
 				flowClient,
 				flow.RoleCollection,
 				*machineAccountInfo,
 			)
+
 			return validator, err
 		}).
 		Component("follower engine", func(builder cmd.NodeBuilder, node *cmd.NodeConfig) (module.ReadyDoneAware, error) {
@@ -475,17 +461,17 @@ func main() {
 			signer := verification.NewSingleSigner(staking, node.Me.NodeID())
 
 			// construct QC contract client
-			qcContractClient, err := createQCContractClient(node, machineAccountInfo, flowClient)
+			qcContractClients, err := createQCContractClients(node, machineAccountInfo, flowClientOpts)
 			if err != nil {
-				return nil, fmt.Errorf("could not create qc contract client %w", err)
+				return nil, fmt.Errorf("could not create qc contract clients %w", err)
 			}
 
 			rootQCVoter := epochs.NewRootQCVoter(
 				node.Logger,
 				node.Me,
 				signer,
 				node.State,
-				qcContractClient,
+				qcContractClients,
 			)
 
 			factory := factories.NewEpochComponentsFactory(
@@ -545,3 +531,23 @@ func createQCContractClient(node *cmd.NodeConfig, machineAccountInfo *bootstrap.
 
 	return qcContractClient, nil
 }
+
+// createQCContractClients creates priority ordered array of QCContractClient
+func createQCContractClients(node *cmd.NodeConfig, machineAccountInfo *bootstrap.NodeMachineAccountInfo, flowClientOpts []*common.FlowClientConfig) ([]module.QCContractClient, error) {
+	qcClients := make([]module.QCContractClient, 0)
+
+	for _, opt := range flowClientOpts {
+		flowClient, err := common.FlowClient(opt)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create flow client for qc contract client with options: %s %w", flowClientOpts, err)
+		}
+
+		qcClient, err := createQCContractClient(node, machineAccountInfo, flowClient)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create qc contract client with flow client options: %s %w", flowClientOpts, err)
+		}
+
+		qcClients = append(qcClients, qcClient)
+	}
+	return qcClients, nil
+}

cmd/consensus/main.go

@@ -95,10 +95,9 @@ func main() {
 
 		// DKG contract client
 		machineAccountInfo *bootstrap.NodeMachineAccountInfo
-		flowClient         *client.Client
-		accessAddress      string
-		secureAccessNodeID string
+		flowClientOpts     []*common.FlowClientConfig
 		insecureAccessAPI  bool
+		accessNodeIDS      []string
 
 		err                     error
 		mutableState            protocol.MutableState
@@ -145,9 +144,8 @@ func main() {
 		flags.UintVar(&requiredApprovalsForSealVerification, "required-verification-seal-approvals", validation.DefaultRequiredApprovalsForSealValidation, "minimum number of approvals that are required to verify a seal")
 		flags.UintVar(&requiredApprovalsForSealConstruction, "required-construction-seal-approvals", sealing.DefaultRequiredApprovalsForSealConstruction, "minimum number of approvals that are required to construct a seal")
 		flags.BoolVar(&emergencySealing, "emergency-sealing-active", sealing.DefaultEmergencySealingActive, "(de)activation of emergency sealing")
-		flags.StringVar(&accessAddress, "access-address", "", "the address of an access node")
-		flags.StringVar(&secureAccessNodeID, "secure-access-node-id", "", "the node ID of the secure access GRPC server")
-		flags.BoolVar(&insecureAccessAPI, "insecure-access-api", true, "required if insecure GRPC connection should be used")
+		flags.BoolVar(&insecureAccessAPI, "insecure-access-api", false, "required if insecure GRPC connection should be used")
+		flags.StringSliceVar(&accessNodeIDS, "access-node-ids", []string{}, fmt.Sprintf("array of access node ID's sorted in priority order where the first ID in this array will get the first connection attempt and each subsequent ID after serves as a fallback. minimum length %d", common.DefaultAccessNodeIDSMinimum))
 		flags.DurationVar(&dkgControllerConfig.BaseStartDelay, "dkg-controller-base-start-delay", dkgmodule.DefaultBaseStartDelay, "used to define the range for jitter prior to DKG start (eg. 500µs) - the base value is scaled quadratically with the # of DKG participants")
 		flags.DurationVar(&dkgControllerConfig.BaseHandleFirstBroadcastDelay, "dkg-controller-base-handle-first-broadcast-delay", dkgmodule.DefaultBaseHandleFirstBroadcastDelay, "used to define the range for jitter prior to DKG handling the first broadcast messages (eg. 50ms) - the base value is scaled quadratically with the # of DKG participants")
 		flags.DurationVar(&dkgControllerConfig.HandleSubsequentBroadcastDelay, "dkg-controller-handle-subsequent-broadcast-delay", dkgmodule.DefaultHandleSubsequentBroadcastDelay, "used to define the constant delay introduced prior to DKG handling subsequent broadcast messages (eg. 2s)")
@@ -368,38 +366,25 @@ func main() {
 			machineAccountInfo, err = cmd.LoadNodeMachineAccountInfoFile(node.BootstrapDir, node.NodeID)
 			return err
 		}).
-		Module("sdk client", func(builder cmd.NodeBuilder, node *cmd.NodeConfig) error {
-			if accessAddress == "" {
-				return fmt.Errorf("missing required flag --access-address")
+		Module("sdk client connection options", func(builder cmd.NodeBuilder, node *cmd.NodeConfig) error {
+			if len(accessNodeIDS) < common.DefaultAccessNodeIDSMinimum {
+				return fmt.Errorf("invalid flag --access-node-ids atleast %d IDs must be provided", common.DefaultAccessNodeIDSMinimum)
 			}
-			// create flow client with correct GRPC configuration for QC contract client
-			if insecureAccessAPI {
-				flowClient, err = common.InsecureFlowClient(accessAddress)
-				return err
-			} else {
-				if secureAccessNodeID == "" {
-					return fmt.Errorf("invalid flag --secure-access-node-id required")
-				}
-
-				nodeID, err := flow.HexStringToIdentifier(secureAccessNodeID)
-				if err != nil {
-					return fmt.Errorf("could not get flow identifer from secured access node id: %s", secureAccessNodeID)
-				}
-
-				identities, err := node.State.Sealed().Identities(filter.HasNodeID(nodeID))
-				if err != nil {
-					return fmt.Errorf("could not get identity of secure access node: %s", secureAccessNodeID)
-				}
-
-				if len(identities) < 1 {
-					return fmt.Errorf("could not find identity of secure access node: %s", secureAccessNodeID)
-				}
 
-				flowClient, err = common.SecureFlowClient(accessAddress, identities[0].NetworkPubKey.String()[2:])
-				return err
+			flowClientOpts, err = common.FlowClientConfigs(accessNodeIDS, insecureAccessAPI, node.State.Sealed())
+			if err != nil {
+				return fmt.Errorf("failed to prepare flow client connection options for each access node id %w", err)
 			}
+
+			return nil
 		}).
 		Component("machine account config validator", func(builder cmd.NodeBuilder, node *cmd.NodeConfig) (module.ReadyDoneAware, error) {
+			//@TODO use fallback logic for flowClient similar to DKG/QC contract clients
+			flowClient, err := common.FlowClient(flowClientOpts[0])
+			if err != nil {
+				return nil, fmt.Errorf("failed to get flow client connection option for access node (0): %s %w", flowClientOpts[0].AccessAddress, err)
+			}
+
 			validator, err := epochs.NewMachineAccountConfigValidator(
 				node.Logger,
 				flowClient,
@@ -745,7 +730,7 @@ func main() {
 			node.ProtocolEvents.AddConsumer(viewsObserver)
 
 			// construct DKG contract client
-			dkgContractClient, err := createDKGContractClient(node, machineAccountInfo, flowClient)
+			dkgContractClients, err := createDKGContractClients(node, machineAccountInfo, flowClientOpts)
 			if err != nil {
 				return nil, fmt.Errorf("could not create dkg contract client %w", err)
 			}
@@ -760,7 +745,7 @@ func main() {
 				dkgmodule.NewControllerFactory(
 					node.Logger,
 					node.Me,
-					dkgContractClient,
+					dkgContractClients,
 					dkgBrokerTunnel,
 					dkgControllerConfig,
 				),
@@ -790,9 +775,8 @@ func loadDKGPrivateData(dir string, myID flow.Identifier) (*dkg.DKGParticipantPr
 	return &priv, nil
 }
 
-// createDKGContractClient creates a DKG contract client
+// createDKGContractClient creates an dkgContractClient
 func createDKGContractClient(node *cmd.NodeConfig, machineAccountInfo *bootstrap.NodeMachineAccountInfo, flowClient *client.Client) (module.DKGContractClient, error) {
-
 	var dkgClient module.DKGContractClient
 
 	contracts, err := systemcontracts.SystemContractsForChain(node.RootChainID)
@@ -820,3 +804,25 @@ func createDKGContractClient(node *cmd.NodeConfig, machineAccountInfo *bootstrap
 
 	return dkgClient, nil
 }
+
+// createDKGContractClients creates an array dkgContractClient that is sorted by retry fallback priority
+func createDKGContractClients(node *cmd.NodeConfig, machineAccountInfo *bootstrap.NodeMachineAccountInfo, flowClientOpts []*common.FlowClientConfig) ([]module.DKGContractClient, error) {
+	dkgClients := make([]module.DKGContractClient, 0)
+
+	for _, opt := range flowClientOpts {
+		flowClient, err := common.FlowClient(opt)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create flow client for dkg contract client with options: %s %w", flowClientOpts, err)
+		}
+
+		node.Logger.Info().Msgf("created dkg contract client with opts: %s", opt.String())
+		dkgClient, err := createDKGContractClient(node, machineAccountInfo, flowClient)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create dkg contract client with flow client options: %s %w", flowClientOpts, err)
+		}
+
+		dkgClients = append(dkgClients, dkgClient)
+	}
+
+	return dkgClients, nil
+}