Skip to content

Commit

Permalink
Merge branch 'master' into yurii/5807-collection-proposal-engine-refa…
Browse files Browse the repository at this point in the history
…ctoring
  • Loading branch information
durkmurder committed Sep 14, 2021
2 parents efaadbe + 386f157 commit 208c3f0
Show file tree
Hide file tree
Showing 19 changed files with 582 additions and 40 deletions.
5 changes: 4 additions & 1 deletion cmd/access/node_builder/access_node_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -472,10 +472,13 @@ func (anb *FlowAccessNodeBuilder) Build() AccessNodeBuilder {

anb.IngestEng, err = ingestion.New(node.Logger, node.Network, node.State, node.Me, anb.RequestEng, node.Storage.Blocks, node.Storage.Headers, node.Storage.Collections, node.Storage.Transactions, node.Storage.Results, node.Storage.Receipts, anb.TransactionMetrics,
anb.CollectionsToMarkFinalized, anb.CollectionsToMarkExecuted, anb.BlocksToMarkExecuted, anb.RpcEng)
if err != nil {
return nil, err
}
anb.RequestEng.WithHandle(anb.IngestEng.OnCollection)
anb.FinalizationDistributor.AddConsumer(anb.IngestEng)

return anb.IngestEng, err
return anb.IngestEng, nil
}).
Component("requester engine", func(builder cmd.NodeBuilder, node *cmd.NodeConfig) (module.ReadyDoneAware, error) {
// We initialize the requester engine inside the ingestion engine due to the mutual dependency. However, in
Expand Down
8 changes: 6 additions & 2 deletions cmd/access/node_builder/staked_access_node_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,9 @@ func (builder *StakedAccessNodeBuilder) enqueueUnstakedNetworkInit(ctx context.C
libP2PFactory, err := builder.initLibP2PFactory(ctx,
builder.NodeID,
builder.NodeConfig.NetworkKey)
builder.MustNot(err)
if err != nil {
return nil, err
}

msgValidators := unstakedNetworkMsgValidators(node.Logger, node.IdentityProvider, builder.NodeID)

Expand All @@ -169,7 +171,9 @@ func (builder *StakedAccessNodeBuilder) enqueueUnstakedNetworkInit(ctx context.C
top := topology.EmptyListTopology{}

network, err := builder.initNetwork(builder.Me, node.Metrics.Network, middleware, top)
builder.MustNot(err)
if err != nil {
return nil, err
}

builder.Network = network
builder.Middleware = middleware
Expand Down
22 changes: 14 additions & 8 deletions cmd/access/node_builder/unstaked_access_node_builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,11 @@ func (anb *UnstakedAccessNodeBuilder) initNodeInfo() {
// use the networking key that has been passed in the config
networkingKey := anb.AccessNodeConfig.NetworkKey
pubKey, err := keyutils.LibP2PPublicKeyFromFlow(networkingKey.PublicKey())
anb.MustNot(err)
anb.MustNot(err).Msg("could not load networking public key")
peerID, err := peer.IDFromPublicKey(pubKey)
anb.MustNot(err)
anb.MustNot(err).Msg("could not get peer ID from public key")
anb.NodeID, err = p2p.NewUnstakedNetworkIDTranslator().GetFlowID(peerID)
anb.MustNot(err)
anb.MustNot(err).Msg("could not get flow node ID")
anb.NodeConfig.NetworkKey = networkingKey // copy the key to NodeConfig
anb.NodeConfig.StakingKey = nil // no staking key for the unstaked node
}
Expand Down Expand Up @@ -105,7 +105,7 @@ func (builder *FlowAccessNodeBuilder) deriveBootstrapPeerIdentities() {
return
}
ids, err := BootstrapIdentities(builder.bootstrapNodeAddresses, builder.bootstrapNodePublicKeys)
builder.MustNot(err)
builder.MustNot(err).Msg("failed to derive bootstrap peer identities")
builder.bootstrapIdentities = ids
}

Expand Down Expand Up @@ -145,7 +145,9 @@ func (builder *UnstakedAccessNodeBuilder) initLibP2PFactory(ctx context.Context,

// seed the DHT with the boostrap identities
bootstrapPeersOpt, err := p2p.WithBootstrapPeers(builder.bootstrapIdentities)
builder.MustNot(err)
if err != nil {
return nil, err
}
dhtOptions = append(dhtOptions, bootstrapPeersOpt)

connManager := p2p.NewConnManager(builder.Logger, builder.Metrics.Network, p2p.TrackUnstakedConnections(builder.IdentityProvider))
Expand Down Expand Up @@ -210,7 +212,9 @@ func (anb *UnstakedAccessNodeBuilder) enqueueMiddleware(ctx context.Context) {
unstakedNetworkMetrics := metrics.NewNoopCollector()

libP2PFactory, err := anb.initLibP2PFactory(ctx, unstakedNodeID, unstakedNetworkKey)
anb.MustNot(err)
if err != nil {
return err
}

msgValidators := unstakedNetworkMsgValidators(node.Logger, node.IdentityProvider, unstakedNodeID)

Expand Down Expand Up @@ -238,7 +242,9 @@ func (anb *UnstakedAccessNodeBuilder) enqueueUnstakedNetworkInit(ctx context.Con

// topology is nil since its automatically managed by libp2p
network, err := anb.initNetwork(anb.Me, unstakedNetworkMetrics, anb.Middleware, nil)
anb.MustNot(err)
if err != nil {
return nil, err
}

anb.Network = converter.NewNetwork(network, engine.SyncCommittee, engine.UnstakedSyncCommittee)

Expand All @@ -247,7 +253,7 @@ func (anb *UnstakedAccessNodeBuilder) enqueueUnstakedNetworkInit(ctx context.Con
idEvents := gadgets.NewIdentityDeltas(anb.Middleware.UpdateNodeAddresses)
anb.ProtocolEvents.AddConsumer(idEvents)

return anb.Network, err
return anb.Network, nil
})
}

Expand Down
101 changes: 101 additions & 0 deletions cmd/bootstrap/cmd/check_machine_account.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
package cmd

import (
"context"
"fmt"
"path/filepath"
"time"

"github.com/spf13/cobra"
"google.golang.org/grpc"

sdk "github.com/onflow/flow-go-sdk"
"github.com/onflow/flow-go-sdk/client"
"github.com/onflow/flow-go/cmd"
model "github.com/onflow/flow-go/model/bootstrap"
"github.com/onflow/flow-go/module/epochs"
)

var (
flagAccessAPIAddress string
)

var checkMachineAccountCmd = &cobra.Command{
Use: "check-machine-account",
Short: "checks a machine account configuration",
Run: checkMachineAccountRun,
}

func init() {
rootCmd.AddCommand(checkMachineAccountCmd)

checkMachineAccountCmd.Flags().StringVar(&flagAccessAPIAddress, "access-address", "", "network address of an Access Node")
cmd.MarkFlagRequired(checkMachineAccountCmd, "access-address")
}

func checkMachineAccountRun(_ *cobra.Command, _ []string) {

// read nodeID written to boostrap dir by `bootstrap key`
nodeID, err := readNodeID()
if err != nil {
log.Fatal().Err(err).Msg("could not read node id")
}

// read the private node information - used to get the role
var nodeInfoPriv model.NodeInfoPriv
readJSON(filepath.Join(flagOutdir, fmt.Sprintf(model.PathNodeInfoPriv, nodeID)), &nodeInfoPriv)

// read the machine account info file
machineAccountInfo := readMachineAccountInfo(nodeID)

machineAccountPrivKey, err := machineAccountInfo.PrivateKey()
if err != nil {
log.Fatal().Err(err).Msg("could not decode machine account private key")
}

// print public machine account info
log.Debug().
Str("machine_account_address", machineAccountInfo.Address).
Str("machine_account_pub_key", fmt.Sprintf("%x", encodedRuntimeAccountPubKey(machineAccountPrivKey))).
Uint("key_index", machineAccountInfo.KeyIndex).
Str("signing_algo", machineAccountInfo.SigningAlgorithm.String()).
Str("hash_algo", machineAccountInfo.HashAlgorithm.String()).
Msg("read machine account info from disk")

flowClient, err := client.New(flagAccessAPIAddress, grpc.WithInsecure())
if err != nil {
log.Fatal().Err(err).Msgf("could not connect to access API at address %s", flagAccessAPIAddress)
}

// retrieve the on-chain machine account info
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
onChainAccount, err := flowClient.GetAccount(ctx, sdk.HexToAddress(machineAccountInfo.Address))
if err != nil {
log.Fatal().Err(err).Msg("could not read account")
}

// check the local machine account config with the on-chain account
// this will log non-critical warnings, and return an error for critical problems
err = epochs.CheckMachineAccountInfo(
log,
nodeInfoPriv.Role,
machineAccountInfo,
onChainAccount,
)
if err != nil {
log.Error().Err(err).Msg("⚠️ machine account is misconfigured")
return
}
log.Info().Msg("🤖 machine account is configured correctly")
}

// readMachineAccountInfo reads the machine account info from disk
func readMachineAccountInfo(nodeID string) model.NodeMachineAccountInfo {
var machineAccountInfo model.NodeMachineAccountInfo

path := filepath.Join(flagOutdir, fmt.Sprintf(model.PathNodeMachineAccountInfoPriv, nodeID))
readJSON(path, &machineAccountInfo)

return machineAccountInfo
}
7 changes: 4 additions & 3 deletions cmd/bootstrap/cmd/final_list.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package cmd
import (
"github.com/spf13/cobra"

"github.com/onflow/flow-go/cmd"
model "github.com/onflow/flow-go/model/bootstrap"
"github.com/onflow/flow-go/model/flow"
)
Expand All @@ -27,15 +28,15 @@ func init() {
func addFinalListFlags() {
// partner node info flag
finalListCmd.Flags().StringVar(&flagPartnerNodeInfoDir, "partner-infos", "", "path to a directory containing all parnter nodes details")
_ = finalListCmd.MarkFlagRequired("partner-infos")
cmd.MarkFlagRequired(finalListCmd, "partner-infos")

// internal/flow node info flag
finalListCmd.Flags().StringVar(&flagInternalNodePrivInfoDir, "flow-infos", "", "path to a directory containing all internal/flow nodes details")
_ = finalListCmd.MarkFlagRequired("flow-infos")
cmd.MarkFlagRequired(finalListCmd, "flow-infos")

// staking nodes dir containing staking nodes json
finalListCmd.Flags().StringVar(&flagStakingNodesPath, "staking-nodes", "", "path to a JSON file of all staking nodes")
_ = finalListCmd.MarkFlagRequired("staking-nodes")
cmd.MarkFlagRequired(finalListCmd, "staking-nodes")

finalListCmd.Flags().UintVar(&flagCollectionClusters, "collection-clusters", 2,
"number of collection clusters")
Expand Down
25 changes: 13 additions & 12 deletions cmd/bootstrap/cmd/finalize.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/onflow/cadence"
"github.com/spf13/cobra"

"github.com/onflow/flow-go/cmd"
"github.com/onflow/flow-go/cmd/bootstrap/run"
"github.com/onflow/flow-go/fvm"
model "github.com/onflow/flow-go/model/bootstrap"
Expand Down Expand Up @@ -76,10 +77,10 @@ func addFinalizeCmdFlags() {
finalizeCmd.Flags().StringVar(&flagPartnerStakes, "partner-stakes", "", "path to a JSON file containing "+
"a map from partner node's NodeID to their stake")

_ = finalizeCmd.MarkFlagRequired("config")
_ = finalizeCmd.MarkFlagRequired("internal-priv-dir")
_ = finalizeCmd.MarkFlagRequired("partner-dir")
_ = finalizeCmd.MarkFlagRequired("partner-stakes")
cmd.MarkFlagRequired(finalizeCmd, "config")
cmd.MarkFlagRequired(finalizeCmd, "internal-priv-dir")
cmd.MarkFlagRequired(finalizeCmd, "partner-dir")
cmd.MarkFlagRequired(finalizeCmd, "partner-stakes")

// required parameters for generation of root block, root execution result and root block seal
finalizeCmd.Flags().StringVar(&flagRootChain, "root-chain", "local", "chain ID for the root block (can be 'main', 'test', 'canary', 'bench', or 'local'")
Expand All @@ -92,14 +93,14 @@ func addFinalizeCmdFlags() {
finalizeCmd.Flags().Uint64Var(&flagNumViewsInStakingAuction, "epoch-staking-phase-length", 100, "length of the epoch staking phase measured in views")
finalizeCmd.Flags().Uint64Var(&flagNumViewsInDKGPhase, "epoch-dkg-phase-length", 1000, "length of each DKG phase measured in views")

_ = finalizeCmd.MarkFlagRequired("root-chain")
_ = finalizeCmd.MarkFlagRequired("root-parent")
_ = finalizeCmd.MarkFlagRequired("root-height")
_ = finalizeCmd.MarkFlagRequired("root-commit")
_ = finalizeCmd.MarkFlagRequired("epoch-counter")
_ = finalizeCmd.MarkFlagRequired("epoch-length")
_ = finalizeCmd.MarkFlagRequired("epoch-staking-phase-length")
_ = finalizeCmd.MarkFlagRequired("epoch-dkg-phase-length")
cmd.MarkFlagRequired(finalizeCmd, "root-chain")
cmd.MarkFlagRequired(finalizeCmd, "root-parent")
cmd.MarkFlagRequired(finalizeCmd, "root-height")
cmd.MarkFlagRequired(finalizeCmd, "root-commit")
cmd.MarkFlagRequired(finalizeCmd, "epoch-counter")
cmd.MarkFlagRequired(finalizeCmd, "epoch-length")
cmd.MarkFlagRequired(finalizeCmd, "epoch-staking-phase-length")
cmd.MarkFlagRequired(finalizeCmd, "epoch-dkg-phase-length")

finalizeCmd.Flags().BytesHexVar(&flagBootstrapRandomSeed, "random-seed", GenerateRandomSeed(), "The seed used to for DKG, Clustering and Cluster QC generation")

Expand Down
5 changes: 3 additions & 2 deletions cmd/bootstrap/cmd/key.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"net"
"strconv"

"github.com/onflow/flow-go/cmd"
"github.com/onflow/flow-go/cmd/bootstrap/utils"

"github.com/multiformats/go-multiaddr"
Expand Down Expand Up @@ -38,9 +39,9 @@ func init() {

// required flags
keyCmd.Flags().StringVar(&flagRole, "role", "", "node role (can be \"collection\", \"consensus\", \"execution\", \"verification\" or \"access\")")
_ = keyCmd.MarkFlagRequired("role")
cmd.MarkFlagRequired(keyCmd, "role")
keyCmd.Flags().StringVar(&flagAddress, "address", "", "network address")
_ = keyCmd.MarkFlagRequired("address")
cmd.MarkFlagRequired(keyCmd, "address")

keyCmd.Flags().BytesHexVar(&flagNetworkSeed, "networking-seed", []byte{}, fmt.Sprintf("hex encoded networking seed (min %d bytes)", minSeedBytes))
keyCmd.Flags().BytesHexVar(&flagStakingSeed, "staking-seed", []byte{}, fmt.Sprintf("hex encoded staking seed (min %d bytes)", minSeedBytes))
Expand Down
3 changes: 2 additions & 1 deletion cmd/bootstrap/cmd/keygen.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"io"
"os"

"github.com/onflow/flow-go/cmd"
"github.com/onflow/flow-go/cmd/bootstrap/utils"

"github.com/spf13/cobra"
Expand Down Expand Up @@ -87,7 +88,7 @@ func init() {

// required parameters
keygenCmd.Flags().StringVar(&flagConfig, "config", "node-config.json", "path to a JSON file containing multiple node configurations (Role, Address, Stake)")
_ = keygenCmd.MarkFlagRequired("config")
cmd.MarkFlagRequired(keygenCmd, "config")

// optional parameters, used for generating machine account files
keygenCmd.Flags().BoolVar(&flagDefaultMachineAccount, "machine-account", false, "whether or not to generate a default (same as networking key) machine account key file")
Expand Down
7 changes: 3 additions & 4 deletions cmd/bootstrap/cmd/keys.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package cmd
import (
"fmt"

sdkcrypto "github.com/onflow/flow-go-sdk/crypto"
"github.com/onflow/flow-go/cmd/bootstrap/utils"
"github.com/onflow/flow-go/crypto/hash"
"github.com/onflow/flow-go/model/flow/order"
Expand Down Expand Up @@ -90,9 +89,9 @@ func assembleNodeMachineAccountInfo(machineKey crypto.PrivateKey, accountAddress
Msg("encoded public machine account key")
machineNodeInfo := model.NodeMachineAccountInfo{
EncodedPrivateKey: machineKey.Encode(),
KeyIndex: 0,
SigningAlgorithm: sdkcrypto.ECDSA_P256,
HashAlgorithm: sdkcrypto.SHA3_256,
KeyIndex: model.DefaultMachineAccountKeyIndex,
SigningAlgorithm: model.DefaultMachineAccountSignAlgo,
HashAlgorithm: model.DefaultMachineAccountHashAlgo,
Address: accountAddress,
}
return machineNodeInfo
Expand Down
3 changes: 2 additions & 1 deletion cmd/bootstrap/cmd/machine_account.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (

"github.com/spf13/cobra"

"github.com/onflow/flow-go/cmd"
"github.com/onflow/flow-go/crypto"
model "github.com/onflow/flow-go/model/bootstrap"
"github.com/onflow/flow-go/model/flow"
Expand All @@ -30,7 +31,7 @@ func init() {
rootCmd.AddCommand(machineAccountCmd)

machineAccountCmd.Flags().StringVar(&flagMachineAccountAddress, "address", "", "the node's machine account address")
_ = machineAccountCmd.MarkFlagRequired("address")
cmd.MarkFlagRequired(machineAccountCmd, "address")
}

// keyCmdRun generate the node staking key, networking key and node information
Expand Down
3 changes: 2 additions & 1 deletion cmd/bootstrap/cmd/pull.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (

"github.com/spf13/cobra"

"github.com/onflow/flow-go/cmd"
"github.com/onflow/flow-go/cmd/bootstrap/gcs"
)

Expand All @@ -33,7 +34,7 @@ func init() {

func addPullCmdFlags() {
pullCmd.Flags().StringVar(&flagNetwork, "network", "", "network name to pull partner node information")
_ = pullCmd.MarkFlagRequired("network")
cmd.MarkFlagRequired(pullCmd, "network")

pullCmd.Flags().StringVar(&flagBucketName, "bucket", "flow-genesis-bootstrap", "google bucket name")
}
Expand Down
15 changes: 15 additions & 0 deletions cmd/flags.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package cmd

import (
"github.com/spf13/cobra"
)

// MarkFlagRequired marks a flag added to a cobra command as required. Panics
// if the flag has not been added to the cobra command (indicates misconfiguration
// or typo).
func MarkFlagRequired(command *cobra.Command, flagName string) {
err := command.MarkFlagRequired(flagName)
if err != nil {
panic("marked unknown flag as required: " + err.Error())
}
}

0 comments on commit 208c3f0

Please sign in to comment.