From 1153e2c374b5a424f2898a3bdb43b7ed01c8b1ae Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Sun, 20 May 2018 17:47:47 -0500 Subject: [PATCH 01/16] sharding: revamp sharding client config to single entrypoint --- cmd/geth/main.go | 3 +- cmd/geth/shardingcmd.go | 47 +++++++---------- sharding/client/client.go | 104 +++++++------------------------------- sharding/client/utils.go | 95 ++++++++++++++++++++++++++++++++++ 4 files changed, 133 insertions(+), 116 deletions(-) create mode 100644 sharding/client/utils.go diff --git a/cmd/geth/main.go b/cmd/geth/main.go index d147cf8f78e..25239247d33 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -168,8 +168,7 @@ func init() { attachCommand, javascriptCommand, // See shardingcmd.go: - notaryClientCommand, - proposerClientCommand, + shardingCommand, // See misccmd.go: makecacheCommand, makedagCommand, diff --git a/cmd/geth/shardingcmd.go b/cmd/geth/shardingcmd.go index db110de25dd..7358adc1f65 100644 --- a/cmd/geth/shardingcmd.go +++ b/cmd/geth/shardingcmd.go @@ -1,46 +1,37 @@ package main import ( - "github.com/ethereum/go-ethereum/sharding/notary" - "github.com/ethereum/go-ethereum/sharding/proposer" - "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/sharding/client" cli "gopkg.in/urfave/cli.v1" ) var ( - notaryClientCommand = cli.Command{ - Action: utils.MigrateFlags(notaryClient), - Name: "sharding-notary", - Aliases: []string{"shard-notary"}, - Usage: "Start a sharding notary client", + shardingCommand = cli.Command{ + Action: utils.MigrateFlags(shardingClient), + Name: "sharding", + Usage: "Start a sharding-enabled node", ArgsUsage: "[endpoint]", Flags: []cli.Flag{utils.DataDirFlag, utils.PasswordFileFlag, utils.NetworkIdFlag, utils.IPCPathFlag, utils.DepositFlag}, Category: "SHARDING COMMANDS", Description: ` -Launches a sharding notary client that connects to a running geth node and submit collations to a Sharding Manager Contract. This feature is a work in progress. -`, - } - proposerClientCommand = cli.Command{ - Action: utils.MigrateFlags(proposerClient), - Name: "sharding-proposer", - Aliases: []string{"shard-proposer"}, - Usage: "Start a sharding proposer client", - ArgsUsage: "[endpoint]", - Flags: []cli.Flag{utils.DataDirFlag, utils.PasswordFileFlag, utils.NetworkIdFlag, utils.IPCPathFlag}, - Category: "SHARDING COMMANDS", - Description: ` -Launches a sharding proposer client that connects to a running geth node and proposes collations to notary node. This feature is a work in progress. +Launches a sharding client that submits collations to a Sharding Manager Contract, handles notary and proposer services, and manages shardp2p connections. This feature is a work in progress. `, } ) -func notaryClient(ctx *cli.Context) error { - c := notary.NewNotary(ctx) - return c.Start() +func shardingClient(ctx *cli.Context) error { + // configures a sharding-enabled node using the cli's context. + shardingNode := client.NewClient(ctx) + return shardingNode.Start() } -func proposerClient(ctx *cli.Context) error { - p := proposer.NewProposer(ctx) - return p.Start() -} +// func notaryClient(ctx *cli.Context) error { +// c := notary.NewNotary(ctx) +// return c.Start() +// } + +// func proposerClient(ctx *cli.Context) error { +// p := proposer.NewProposer(ctx) +// return p.Start() +// } diff --git a/sharding/client/client.go b/sharding/client/client.go index 769ce50dd94..a92d097b58d 100644 --- a/sharding/client/client.go +++ b/sharding/client/client.go @@ -4,13 +4,9 @@ package client import ( - "bufio" "context" - "errors" "fmt" "math/big" - "os" - "time" ethereum "github.com/ethereum/go-ethereum" @@ -21,10 +17,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/sharding" "github.com/ethereum/go-ethereum/sharding/contracts" cli "gopkg.in/urfave/cli.v1" ) @@ -33,7 +27,8 @@ const ( clientIdentifier = "geth" // Used to determine the ipc name. ) -// General client for Notary/Proposer - Communicates to Geth node via JSON RPC. +// General client for a sharding-enabled system. +// Communicates to Geth node via JSON RPC. type shardingClient struct { endpoint string // Endpoint to JSON RPC. client *ethclient.Client // Ethereum RPC client. @@ -43,7 +38,7 @@ type shardingClient struct { rpcClient *rpc.Client // The RPC client connection to the main geth node. } -// Client methods that must be implemented to run a sharded system. +// Client methods that must be implemented to run a sharding node. type Client interface { Start() error Close() @@ -55,7 +50,8 @@ type Client interface { DepositFlagSet() bool } -// NewClient forms a new struct instance. +// NewClient setups the sharding config, registers the services required +// by the sharded system. func NewClient(ctx *cli.Context) Client { path := node.DefaultDataDir() if ctx.GlobalIsSet(utils.DataDirFlag.Name) { @@ -80,11 +76,22 @@ func NewClient(ctx *cli.Context) Client { } ks := keystore.NewKeyStore(keydir, scryptN, scryptP) - return &shardingClient{ + // Registers required services. Notary/Proposer are services in a sharding client, + // and they are selected based on command line flags at runtime. + + c := &shardingClient{ endpoint: endpoint, keystore: ks, ctx: ctx, } + if err := c.registerShardingServices(); err != nil { + panic(err) // TODO(rauljordan): handle this. + } + return c +} + +func (c *shardingClient) registerShardingServices() error { + return nil } // Start the sharding client. @@ -104,7 +111,7 @@ func (c *shardingClient) Start() error { return fmt.Errorf("no accounts found") } - if err := c.unlockAccount(accounts[0]); err != nil { + if err := unlockAccount(c, accounts[0]); err != nil { return fmt.Errorf("cannot unlock account. %v", err) } @@ -122,31 +129,6 @@ func (c *shardingClient) Close() { c.rpcClient.Close() } -// UnlockAccount will unlock the specified account using utils.PasswordFileFlag or empty string if unset. -func (c *shardingClient) unlockAccount(account accounts.Account) error { - pass := "" - - if c.ctx.GlobalIsSet(utils.PasswordFileFlag.Name) { - file, err := os.Open(c.ctx.GlobalString(utils.PasswordFileFlag.Name)) - if err != nil { - return fmt.Errorf("unable to open file containing account password %s. %v", utils.PasswordFileFlag.Value, err) - } - scanner := bufio.NewScanner(file) - scanner.Split(bufio.ScanWords) - if !scanner.Scan() { - err = scanner.Err() - if err != nil { - return fmt.Errorf("unable to read contents of file %v", err) - } - return errors.New("password not found in file") - } - - pass = scanner.Text() - } - - return c.keystore.Unlock(account, pass) -} - // CreateTXOpts creates a *TransactOpts with a signer using the default account on the keystore. func (c *shardingClient) CreateTXOpts(value *big.Int) (*bind.TransactOpts, error) { account := c.Account() @@ -191,56 +173,6 @@ func (c *shardingClient) SMCTransactor() *contracts.SMCTransactor { return &c.smc.SMCTransactor } -// dialRPC endpoint to node. -func dialRPC(endpoint string) (*rpc.Client, error) { - if endpoint == "" { - endpoint = node.DefaultIPCEndpoint(clientIdentifier) - } - return rpc.Dial(endpoint) -} - -// initSMC initializes the sharding manager contract bindings. -// If the SMC does not exist, it will be deployed. -func initSMC(c *shardingClient) (*contracts.SMC, error) { - b, err := c.client.CodeAt(context.Background(), sharding.ShardingManagerAddress, nil) - if err != nil { - return nil, fmt.Errorf("unable to get contract code at %s: %v", sharding.ShardingManagerAddress, err) - } - - // Deploy SMC for development only. - // TODO: Separate contract deployment from the sharding client. It would only need to be deployed - // once on the mainnet, so this code would not need to ship with the client. - if len(b) == 0 { - log.Info(fmt.Sprintf("No sharding manager contract found at %s. Deploying new contract.", sharding.ShardingManagerAddress.String())) - - txOps, err := c.CreateTXOpts(big.NewInt(0)) - if err != nil { - return nil, fmt.Errorf("unable to intiate the transaction: %v", err) - } - - addr, tx, contract, err := contracts.DeploySMC(txOps, c.client) - if err != nil { - return nil, fmt.Errorf("unable to deploy sharding manager contract: %v", err) - } - - for pending := true; pending; _, pending, err = c.client.TransactionByHash(context.Background(), tx.Hash()) { - if err != nil { - return nil, fmt.Errorf("unable to get transaction by hash: %v", err) - } - time.Sleep(1 * time.Second) - } - - log.Info(fmt.Sprintf("New contract deployed at %s", addr.String())) - return contract, nil - } - - contract, err := contracts.NewSMC(sharding.ShardingManagerAddress, c.client) - if err != nil { - - } - return contract, nil -} - // DepositFlagSet returns true for cli flag --deposit. func (c *shardingClient) DepositFlagSet() bool { return c.ctx.GlobalBool(utils.DepositFlag.Name) diff --git a/sharding/client/utils.go b/sharding/client/utils.go new file mode 100644 index 00000000000..ce031ed35b0 --- /dev/null +++ b/sharding/client/utils.go @@ -0,0 +1,95 @@ +package client + +import ( + "bufio" + "context" + "errors" + "fmt" + "math/big" + "os" + "time" + + "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/sharding" + "github.com/ethereum/go-ethereum/sharding/contracts" +) + +// dialRPC endpoint to node. +func dialRPC(endpoint string) (*rpc.Client, error) { + if endpoint == "" { + endpoint = node.DefaultIPCEndpoint(clientIdentifier) + } + return rpc.Dial(endpoint) +} + +// initSMC initializes the sharding manager contract bindings. +// If the SMC does not exist, it will be deployed. +func initSMC(c *shardingClient) (*contracts.SMC, error) { + b, err := c.client.CodeAt(context.Background(), sharding.ShardingManagerAddress, nil) + if err != nil { + return nil, fmt.Errorf("unable to get contract code at %s: %v", sharding.ShardingManagerAddress, err) + } + + // Deploy SMC for development only. + // TODO: Separate contract deployment from the sharding client. It would only need to be deployed + // once on the mainnet, so this code would not need to ship with the client. + if len(b) == 0 { + log.Info(fmt.Sprintf("No sharding manager contract found at %s. Deploying new contract.", sharding.ShardingManagerAddress.String())) + + txOps, err := c.CreateTXOpts(big.NewInt(0)) + if err != nil { + return nil, fmt.Errorf("unable to intiate the transaction: %v", err) + } + + addr, tx, contract, err := contracts.DeploySMC(txOps, c.client) + if err != nil { + return nil, fmt.Errorf("unable to deploy sharding manager contract: %v", err) + } + + for pending := true; pending; _, pending, err = c.client.TransactionByHash(context.Background(), tx.Hash()) { + if err != nil { + return nil, fmt.Errorf("unable to get transaction by hash: %v", err) + } + time.Sleep(1 * time.Second) + } + + log.Info(fmt.Sprintf("New contract deployed at %s", addr.String())) + return contract, nil + } + + contract, err := contracts.NewSMC(sharding.ShardingManagerAddress, c.client) + if err != nil { + + } + return contract, nil +} + +// unlockAccount will unlock the specified account using utils.PasswordFileFlag +// or empty string if unset. +func unlockAccount(c *shardingClient, account accounts.Account) error { + pass := "" + + if c.ctx.GlobalIsSet(utils.PasswordFileFlag.Name) { + file, err := os.Open(c.ctx.GlobalString(utils.PasswordFileFlag.Name)) + if err != nil { + return fmt.Errorf("unable to open file containing account password %s. %v", utils.PasswordFileFlag.Value, err) + } + scanner := bufio.NewScanner(file) + scanner.Split(bufio.ScanWords) + if !scanner.Scan() { + err = scanner.Err() + if err != nil { + return fmt.Errorf("unable to read contents of file %v", err) + } + return errors.New("password not found in file") + } + + pass = scanner.Text() + } + + return c.keystore.Unlock(account, pass) +} From ba9e6dc4423c4ce77c4bea277b6e64d905c3172c Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Sun, 20 May 2018 17:54:49 -0500 Subject: [PATCH 02/16] sharding: simplified newclient func --- sharding/client/client.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/sharding/client/client.go b/sharding/client/client.go index a92d097b58d..9408b987c66 100644 --- a/sharding/client/client.go +++ b/sharding/client/client.go @@ -76,21 +76,36 @@ func NewClient(ctx *cli.Context) Client { } ks := keystore.NewKeyStore(keydir, scryptN, scryptP) - // Registers required services. Notary/Proposer are services in a sharding client, - // and they are selected based on command line flags at runtime. - c := &shardingClient{ endpoint: endpoint, keystore: ks, ctx: ctx, } + + // Sets up all the config options for the sharding client based on cli flags. + if err := c.configClient(); err != nil { + panic(err) // TODO(rauljordan): handle this. + } + + // Registers all required services the sharding client will run upon start. + // These include shardp2p servers, notary/proposer event loops, and more. if err := c.registerShardingServices(); err != nil { panic(err) // TODO(rauljordan): handle this. } return c } +func (c *shardingClient) configClient() error { + return nil +} + func (c *shardingClient) registerShardingServices() error { + // Registers either a notary or proposer sharding service dependent on + // the ClientType cli flag. + + // Registers the shardp2p service. + + // Registers the SMC interactions service. return nil } From 184bb5cc7a803b2b3e96876d7613d20fe2c06c6f Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 22 May 2018 06:16:57 -0500 Subject: [PATCH 03/16] sharding: rename client package to node --- cmd/utils/flags.go | 6 +- sharding/client/client.go | 194 ----------------------- sharding/node/node.go | 245 +++++++++++++++++++++++++++++ sharding/{client => node}/utils.go | 49 ++---- sharding/service.go | 16 ++ 5 files changed, 275 insertions(+), 235 deletions(-) delete mode 100644 sharding/client/client.go create mode 100644 sharding/node/node.go rename sharding/{client => node}/utils.go (56%) create mode 100644 sharding/service.go diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 8102cc89038..b4a3897ba05 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -534,11 +534,15 @@ var ( Value: whisper.DefaultMinimumPoW, } - //Sharding Settings + // Sharding Settings DepositFlag = cli.BoolFlag{ Name: "deposit", Usage: "To become a notary with your sharding client, " + new(big.Int).Div(sharding.NotaryDeposit, new(big.Int).Exp(big.NewInt(10), big.NewInt(18), nil)).String() + " ETH will be deposited into SMC", } + ClientType = cli.StringFlag{ + Name: "client-type", + Usage: "notary | proposer", + } ) // MakeDataDir retrieves the currently requested data directory, terminating diff --git a/sharding/client/client.go b/sharding/client/client.go deleted file mode 100644 index 9408b987c66..00000000000 --- a/sharding/client/client.go +++ /dev/null @@ -1,194 +0,0 @@ -// Package client provides an interface for interacting with a running ethereum full node. -// As part of the initial phases of sharding, actors will need access to the sharding management -// contract on the main PoW chain. -package client - -import ( - "context" - "fmt" - "math/big" - - ethereum "github.com/ethereum/go-ethereum" - - "github.com/ethereum/go-ethereum/accounts" - "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/accounts/keystore" - "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethclient" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/sharding/contracts" - cli "gopkg.in/urfave/cli.v1" -) - -const ( - clientIdentifier = "geth" // Used to determine the ipc name. -) - -// General client for a sharding-enabled system. -// Communicates to Geth node via JSON RPC. -type shardingClient struct { - endpoint string // Endpoint to JSON RPC. - client *ethclient.Client // Ethereum RPC client. - keystore *keystore.KeyStore // Keystore containing the single signer. - ctx *cli.Context // Command line context. - smc *contracts.SMC // The deployed sharding management contract. - rpcClient *rpc.Client // The RPC client connection to the main geth node. -} - -// Client methods that must be implemented to run a sharding node. -type Client interface { - Start() error - Close() - CreateTXOpts(*big.Int) (*bind.TransactOpts, error) - ChainReader() ethereum.ChainReader - Account() *accounts.Account - SMCCaller() *contracts.SMCCaller - SMCTransactor() *contracts.SMCTransactor - DepositFlagSet() bool -} - -// NewClient setups the sharding config, registers the services required -// by the sharded system. -func NewClient(ctx *cli.Context) Client { - path := node.DefaultDataDir() - if ctx.GlobalIsSet(utils.DataDirFlag.Name) { - path = ctx.GlobalString(utils.DataDirFlag.Name) - } - - endpoint := ctx.Args().First() - if endpoint == "" { - endpoint = fmt.Sprintf("%s/%s.ipc", path, clientIdentifier) - } - if ctx.GlobalIsSet(utils.IPCPathFlag.Name) { - endpoint = ctx.GlobalString(utils.IPCPathFlag.Name) - } - - config := &node.Config{ - DataDir: path, - } - - scryptN, scryptP, keydir, err := config.AccountConfig() - if err != nil { - panic(err) // TODO(prestonvanloon): handle this. - } - ks := keystore.NewKeyStore(keydir, scryptN, scryptP) - - c := &shardingClient{ - endpoint: endpoint, - keystore: ks, - ctx: ctx, - } - - // Sets up all the config options for the sharding client based on cli flags. - if err := c.configClient(); err != nil { - panic(err) // TODO(rauljordan): handle this. - } - - // Registers all required services the sharding client will run upon start. - // These include shardp2p servers, notary/proposer event loops, and more. - if err := c.registerShardingServices(); err != nil { - panic(err) // TODO(rauljordan): handle this. - } - return c -} - -func (c *shardingClient) configClient() error { - return nil -} - -func (c *shardingClient) registerShardingServices() error { - // Registers either a notary or proposer sharding service dependent on - // the ClientType cli flag. - - // Registers the shardp2p service. - - // Registers the SMC interactions service. - return nil -} - -// Start the sharding client. -// Connects to Geth node. -// Verifies or deploys the sharding manager contract. -func (c *shardingClient) Start() error { - rpcClient, err := dialRPC(c.endpoint) - if err != nil { - return fmt.Errorf("cannot start rpc client. %v", err) - } - c.rpcClient = rpcClient - c.client = ethclient.NewClient(rpcClient) - - // Check account existence and unlock account before starting notary client. - accounts := c.keystore.Accounts() - if len(accounts) == 0 { - return fmt.Errorf("no accounts found") - } - - if err := unlockAccount(c, accounts[0]); err != nil { - return fmt.Errorf("cannot unlock account. %v", err) - } - - smc, err := initSMC(c) - if err != nil { - return err - } - c.smc = smc - - return nil -} - -// Close the RPC client connection. -func (c *shardingClient) Close() { - c.rpcClient.Close() -} - -// CreateTXOpts creates a *TransactOpts with a signer using the default account on the keystore. -func (c *shardingClient) CreateTXOpts(value *big.Int) (*bind.TransactOpts, error) { - account := c.Account() - - return &bind.TransactOpts{ - From: account.Address, - Value: value, - Signer: func(signer types.Signer, addr common.Address, tx *types.Transaction) (*types.Transaction, error) { - networkID, err := c.client.NetworkID(context.Background()) - if err != nil { - return nil, fmt.Errorf("unable to fetch networkID: %v", err) - } - return c.keystore.SignTx(*account, tx, networkID /* chainID */) - }, - }, nil -} - -// Account to use for sharding transactions. -func (c *shardingClient) Account() *accounts.Account { - accounts := c.keystore.Accounts() - - return &accounts[0] -} - -// ChainReader for interacting with the chain. -func (c *shardingClient) ChainReader() ethereum.ChainReader { - return ethereum.ChainReader(c.client) -} - -// Client to interact with ethereum node. -func (c *shardingClient) ethereumClient() *ethclient.Client { - return c.client -} - -// SMCCaller to interact with the sharding manager contract. -func (c *shardingClient) SMCCaller() *contracts.SMCCaller { - return &c.smc.SMCCaller -} - -// SMCTransactor allows us to send tx's to the SMC programmatically. -func (c *shardingClient) SMCTransactor() *contracts.SMCTransactor { - return &c.smc.SMCTransactor -} - -// DepositFlagSet returns true for cli flag --deposit. -func (c *shardingClient) DepositFlagSet() bool { - return c.ctx.GlobalBool(utils.DepositFlag.Name) -} diff --git a/sharding/node/node.go b/sharding/node/node.go new file mode 100644 index 00000000000..a1995d50609 --- /dev/null +++ b/sharding/node/node.go @@ -0,0 +1,245 @@ +// Package sharding/node provides an interface for interacting with a running ethereum full node. +// As part of the initial phases of sharding, actors will need access to the sharding management +// contract on the main PoW chain. +package node + +import ( + "bufio" + "context" + "errors" + "fmt" + "math/big" + "os" + + ethereum "github.com/ethereum/go-ethereum" + + "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/keystore" + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/sharding/contracts" + "github.com/ethereum/go-ethereum/sharding/notary" + "github.com/ethereum/go-ethereum/sharding/proposer" + cli "gopkg.in/urfave/cli.v1" +) + +const ( + clientIdentifier = "geth" // Used to determine the ipc name. +) + +// General node for a sharding-enabled system. +// Communicates to Geth node via JSON RPC. +type shardingNode struct { + endpoint string // Endpoint to JSON RPC. + client *ethclient.Client // Ethereum RPC client. + keystore *keystore.KeyStore // Keystore containing the single signer. + ctx *cli.Context // Command line context. + smc *contracts.SMC // The deployed sharding management contract. + rpcClient *rpc.Client // The RPC client connection to the main geth node. +} + +// Node methods that must be implemented to run a sharding node. +type Node interface { + Start() error + Close() + CreateTXOpts(*big.Int) (*bind.TransactOpts, error) + ChainReader() ethereum.ChainReader + Account() *accounts.Account + SMCCaller() *contracts.SMCCaller + SMCTransactor() *contracts.SMCTransactor + DepositFlagSet() bool +} + +// NewCNode setups the sharding config, registers the services required +// by the sharded system. +func NewNode(ctx *cli.Context) Client { + c := &shardingNode{ctx: ctx} + + // Sets up all configuration options based on cli flags. + if err := c.configShardingNode(); err != nil { + panic(err) // TODO(rauljordan): handle this + } + + // Registers all required services the sharding node will run upon start. + // These include shardp2p servers, notary/proposer event loops, and more. + if err := c.registerShardingServices(); err != nil { + panic(err) // TODO(rauljordan): handle this. + } + return c +} + +// configShardingNode uses cli flags to configure the data +// directory, ipc endpoints, keystores, and more. +func (n *shardingNode) configShardingNode() error { + path := node.DefaultDataDir() + if n.ctx.GlobalIsSet(utils.DataDirFlag.Name) { + path = n.ctx.GlobalString(utils.DataDirFlag.Name) + } + + endpoint := n.ctx.Args().First() + if endpoint == "" { + endpoint = fmt.Sprintf("%s/%s.ipc", path, clientIdentifier) + } + if n.ctx.GlobalIsSet(utils.IPCPathFlag.Name) { + endpoint = n.ctx.GlobalString(utils.IPCPathFlag.Name) + } + + config := &node.Config{ + DataDir: path, + } + + scryptN, scryptP, keydir, err := config.AccountConfig() + if err != nil { + panic(err) // TODO(prestonvanloon): handle this. + } + + ks := keystore.NewKeyStore(keydir, scryptN, scryptP) + + n.endpoint = endpoint + n.keystore = ks + + return nil +} + +// registerShardingServices sets up either a notary or proposer +// sharding service dependent on the ClientType cli flag +func (n *shardingNode) registerShardingServices() error { + clientType := n.ctx.GlobalString(utils.ClientType.Name) + + err := n.Register(func(ctx *node.ServiceContext) (node.Service, error) { + if clientType == "notary" { + return notary.NewNotary(n.ctx) + } + return proposer.NewProposer(n.ctx) + }) + + if err != nil { + return fmt.Errorf("failed to register the main sharding services: %v", err) + } + // TODO: registers the shardp2p service. + + return nil +} + +// Register appends a struct to the sharding node's services that +// satisfies the Service interface containing lifecycle handlers. Notary, Proposer, +// and ShardP2P are examples of services. The rationale behind this is that the +// sharding node should know very little about the functioning of its underlying +// services as they should be extensible. +func (n *shardingNode) Register(a interface{}) error { + return nil +} + +// Start is the main entrypoint of a sharding node. It starts off every service +// attached to it. +func (n *shardingNode Start() error { + rpcClient, err := dialRPC(c.endpoint) + if err != nil { + return fmt.Errorf("cannot start rpc client. %v", err) + } + n.rpcClient = rpcClient + n.client = ethclient.NewClient(rpcClient) + + // Check account existence and unlock account before starting notary client. + accounts := n.keystore.Accounts() + if len(accounts) == 0 { + return fmt.Errorf("no accounts found") + } + + if err := n.unlockAccount(accounts[0]); err != nil { + return fmt.Errorf("cannot unlock account. %v", err) + } + + smc, err := initSMC(c) + if err != nil { + return err + } + n.smc = smc + + return nil +} + +// Close the RPC client connection. +func (n *shardingNode) Close() { + n.rpcClient.Close() +} + +// CreateTXOpts creates a *TransactOpts with a signer using the default account on the keystore. +func (n *shardingNode) CreateTXOpts(value *big.Int) (*bind.TransactOpts, error) { + account := n.Account() + + return &bind.TransactOpts{ + From: account.Address, + Value: value, + Signer: func(signer types.Signer, addr common.Address, tx *types.Transaction) (*types.Transaction, error) { + networkID, err := n.client.NetworkID(context.Background()) + if err != nil { + return nil, fmt.Errorf("unable to fetch networkID: %v", err) + } + return n.keystore.SignTx(*account, tx, networkID /* chainID */) + }, + }, nil +} + +// Account to use for sharding transactions. +func (n *shardingNode) Account() *accounts.Account { + accounts := n.keystore.Accounts() + + return &accounts[0] +} + +// ChainReader for interacting with the chain. +func (n *shardingNode) ChainReader() ethereum.ChainReader { + return ethereum.ChainReader(n.client) +} + +// SMCCaller to interact with the sharding manager contract. +func (n *shardingNode) SMCCaller() *contracts.SMCCaller { + return &n.smc.SMCCaller +} + +// SMCTransactor allows us to send tx's to the SMC programmatically. +func (n *shardingNode) SMCTransactor() *contracts.SMCTransactor { + return &n.smc.SMCTransactor +} + +// DepositFlagSet returns true for cli flag --deposit. +func (n *shardingNode) DepositFlagSet() bool { + return n.ctx.GlobalBool(utils.DepositFlag.Name) +} + +// Client to interact with a geth node via JSON-RPC. +func (n *shardingNode) ethereumClient() *ethclient.Client { + return n.client +} + +// unlockAccount will unlock the specified account using utils.PasswordFileFlag +// or empty string if unset. +func (n *shardingNode) unlockAccount(account accounts.Account) error { + pass := "" + + if n.ctx.GlobalIsSet(utils.PasswordFileFlag.Name) { + file, err := os.Open(n.ctx.GlobalString(utils.PasswordFileFlag.Name)) + if err != nil { + return fmt.Errorf("unable to open file containing account password %s. %v", utils.PasswordFileFlag.Value, err) + } + scanner := bufio.NewScanner(file) + scanner.Split(bufio.ScanWords) + if !scanner.Scan() { + err = scanner.Err() + if err != nil { + return fmt.Errorf("unable to read contents of file %v", err) + } + return errors.New("password not found in file") + } + + pass = scanner.Text() + } + + return n.keystore.Unlock(account, pass) +} diff --git a/sharding/client/utils.go b/sharding/node/utils.go similarity index 56% rename from sharding/client/utils.go rename to sharding/node/utils.go index ce031ed35b0..61503259d90 100644 --- a/sharding/client/utils.go +++ b/sharding/node/utils.go @@ -1,16 +1,11 @@ -package client +package node import ( - "bufio" "context" - "errors" "fmt" "math/big" - "os" "time" - "github.com/ethereum/go-ethereum/accounts" - "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" @@ -28,19 +23,19 @@ func dialRPC(endpoint string) (*rpc.Client, error) { // initSMC initializes the sharding manager contract bindings. // If the SMC does not exist, it will be deployed. -func initSMC(c *shardingClient) (*contracts.SMC, error) { - b, err := c.client.CodeAt(context.Background(), sharding.ShardingManagerAddress, nil) +func initSMC(n *shardingNode) (*contracts.SMC, error) { + b, err := n.client.CodeAt(context.Background(), sharding.ShardingManagerAddress, nil) if err != nil { return nil, fmt.Errorf("unable to get contract code at %s: %v", sharding.ShardingManagerAddress, err) } // Deploy SMC for development only. - // TODO: Separate contract deployment from the sharding client. It would only need to be deployed - // once on the mainnet, so this code would not need to ship with the client. + // TODO: Separate contract deployment from the sharding node. It would only need to be deployed + // once on the mainnet, so this code would not need to ship with the node. if len(b) == 0 { log.Info(fmt.Sprintf("No sharding manager contract found at %s. Deploying new contract.", sharding.ShardingManagerAddress.String())) - txOps, err := c.CreateTXOpts(big.NewInt(0)) + txOps, err := n.CreateTXOpts(big.NewInt(0)) if err != nil { return nil, fmt.Errorf("unable to intiate the transaction: %v", err) } @@ -50,7 +45,7 @@ func initSMC(c *shardingClient) (*contracts.SMC, error) { return nil, fmt.Errorf("unable to deploy sharding manager contract: %v", err) } - for pending := true; pending; _, pending, err = c.client.TransactionByHash(context.Background(), tx.Hash()) { + for pending := true; pending; _, pending, err = n.client.TransactionByHash(context.Background(), tx.Hash()) { if err != nil { return nil, fmt.Errorf("unable to get transaction by hash: %v", err) } @@ -61,35 +56,9 @@ func initSMC(c *shardingClient) (*contracts.SMC, error) { return contract, nil } - contract, err := contracts.NewSMC(sharding.ShardingManagerAddress, c.client) + contract, err := contracts.NewSMC(sharding.ShardingManagerAddress, n.client) if err != nil { - + // TODO(terenc3t): handle this } return contract, nil } - -// unlockAccount will unlock the specified account using utils.PasswordFileFlag -// or empty string if unset. -func unlockAccount(c *shardingClient, account accounts.Account) error { - pass := "" - - if c.ctx.GlobalIsSet(utils.PasswordFileFlag.Name) { - file, err := os.Open(c.ctx.GlobalString(utils.PasswordFileFlag.Name)) - if err != nil { - return fmt.Errorf("unable to open file containing account password %s. %v", utils.PasswordFileFlag.Value, err) - } - scanner := bufio.NewScanner(file) - scanner.Split(bufio.ScanWords) - if !scanner.Scan() { - err = scanner.Err() - if err != nil { - return fmt.Errorf("unable to read contents of file %v", err) - } - return errors.New("password not found in file") - } - - pass = scanner.Text() - } - - return c.keystore.Unlock(account, pass) -} diff --git a/sharding/service.go b/sharding/service.go new file mode 100644 index 00000000000..389a6122240 --- /dev/null +++ b/sharding/service.go @@ -0,0 +1,16 @@ +package sharding + +// Service defines items that can be registered into a sharding client. +// +// life-cycle management is delegated to the sharding client. The service is allowed to +// initialize itself upon creation, but no goroutines should be spun up outside of the +// Start method. +type Service interface { + // Start is called after all services have been constructed and the networking + // layer was also initialized to spawn any goroutines required by the service. + Start() error + + // Stop terminates all goroutines belonging to the service, blocking until they + // are all terminated. + Stop() error +} From 4e19d2e96d94677e9d2347f3ae2435e8c2aa41b4 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 22 May 2018 06:34:12 -0500 Subject: [PATCH 04/16] sharding: refactor proposers --- sharding/node/node.go | 1 + sharding/proposer/proposer.go | 29 +++++++++++++++++++++ sharding/proposer/proposer_client.go | 38 ---------------------------- 3 files changed, 30 insertions(+), 38 deletions(-) create mode 100644 sharding/proposer/proposer.go delete mode 100644 sharding/proposer/proposer_client.go diff --git a/sharding/node/node.go b/sharding/node/node.go index a1995d50609..f650e15ed04 100644 --- a/sharding/node/node.go +++ b/sharding/node/node.go @@ -112,6 +112,7 @@ func (n *shardingNode) registerShardingServices() error { clientType := n.ctx.GlobalString(utils.ClientType.Name) err := n.Register(func(ctx *node.ServiceContext) (node.Service, error) { + // TODO(terenc3t): handle case when we just want to start an observer node. if clientType == "notary" { return notary.NewNotary(n.ctx) } diff --git a/sharding/proposer/proposer.go b/sharding/proposer/proposer.go new file mode 100644 index 00000000000..d7aa1dba44f --- /dev/null +++ b/sharding/proposer/proposer.go @@ -0,0 +1,29 @@ +package proposer + +import ( + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/sharding/node" + "github.com/ethereum/go-ethereum/swarm/api/client" + cli "gopkg.in/urfave/cli.v1" +) + +// Proposer holds functionality required to run a collation proposer +// in a sharded system. Must satisfy the Service interface defined in +// sharding/service.go. +type Proposer struct { + client client.Client +} + +// NewProposer creates a struct instance. It is initialized and +// registered as a service upon start of a sharding node. +// Has access to the public methods of this node. +func NewProposer(ctx *cli.Context, node node.Node) *Proposer { + return &Proposer{} +} + +// Start the main entry point for proposing collations. +func (p *Proposer) Start() error { + log.Info("Starting proposer client") + // TODO: Propose collations. + return nil +} diff --git a/sharding/proposer/proposer_client.go b/sharding/proposer/proposer_client.go deleted file mode 100644 index e009b4d4fd6..00000000000 --- a/sharding/proposer/proposer_client.go +++ /dev/null @@ -1,38 +0,0 @@ -package proposer - -import ( - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/sharding/client" - cli "gopkg.in/urfave/cli.v1" -) - -// Proposer holds functionality required to run a collation proposer -// in a sharded system. -type Proposer interface { - Start() error -} - -type proposer struct { - client client.Client -} - -// NewProposer creates a struct instance. -func NewProposer(ctx *cli.Context) Proposer { - return &proposer{ - client: client.NewClient(ctx), - } -} - -// Start the main entry point for proposing collations. -func (p *proposer) Start() error { - log.Info("Starting proposer client") - err := p.client.Start() - if err != nil { - return err - } - defer p.client.Close() - - // TODO: Propose collations. - - return nil -} From 93baa1f22c4b73333313aef9b0c6a11dce6328cc Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 22 May 2018 06:53:15 -0500 Subject: [PATCH 05/16] sharding: tests pass --- sharding/node/node.go | 12 ------ sharding/notary/notary.go | 41 +++++++++--------- sharding/notary/notary_client.go | 42 ------------------- sharding/notary/service.go | 38 +++++++++++++++++ .../{notary_test.go => service_test.go} | 38 +++++++++-------- sharding/proposer/{proposer.go => service.go} | 9 ++-- sharding/{service.go => types.go} | 21 ++++++++++ 7 files changed, 103 insertions(+), 98 deletions(-) delete mode 100644 sharding/notary/notary_client.go create mode 100644 sharding/notary/service.go rename sharding/notary/{notary_test.go => service_test.go} (81%) rename sharding/proposer/{proposer.go => service.go} (75%) rename sharding/{service.go => types.go} (52%) diff --git a/sharding/node/node.go b/sharding/node/node.go index f650e15ed04..0aacaf43c88 100644 --- a/sharding/node/node.go +++ b/sharding/node/node.go @@ -43,18 +43,6 @@ type shardingNode struct { rpcClient *rpc.Client // The RPC client connection to the main geth node. } -// Node methods that must be implemented to run a sharding node. -type Node interface { - Start() error - Close() - CreateTXOpts(*big.Int) (*bind.TransactOpts, error) - ChainReader() ethereum.ChainReader - Account() *accounts.Account - SMCCaller() *contracts.SMCCaller - SMCTransactor() *contracts.SMCTransactor - DepositFlagSet() bool -} - // NewCNode setups the sharding config, registers the services required // by the sharded system. func NewNode(ctx *cli.Context) Client { diff --git a/sharding/notary/notary.go b/sharding/notary/notary.go index 9d115b48844..a96de82afa0 100644 --- a/sharding/notary/notary.go +++ b/sharding/notary/notary.go @@ -11,7 +11,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/sharding" - "github.com/ethereum/go-ethereum/sharding/client" ) // SubscribeBlockHeaders checks incoming block headers and determines if @@ -19,10 +18,10 @@ import ( // from the running geth node and sorts them by descending order of gas price, // eliminates those that ask for too much gas, and routes them over // to the SMC to create a collation. -func subscribeBlockHeaders(c client.Client) error { +func subscribeBlockHeaders(n sharding.Node) error { headerChan := make(chan *types.Header, 16) - _, err := c.ChainReader().SubscribeNewHead(context.Background(), headerChan) + _, err := n.ChainReader().SubscribeNewHead(context.Background(), headerChan) if err != nil { return fmt.Errorf("unable to subscribe to incoming headers. %v", err) } @@ -36,13 +35,13 @@ func subscribeBlockHeaders(c client.Client) error { log.Info(fmt.Sprintf("Received new header: %v", head.Number.String())) // Check if we are in the notary pool before checking if we are an eligible notary. - v, err := isAccountInNotaryPool(c) + v, err := isAccountInNotaryPool(n) if err != nil { return fmt.Errorf("unable to verify client in notary pool. %v", err) } if v { - if err := checkSMCForNotary(c, head); err != nil { + if err := checkSMCForNotary(n, head); err != nil { return fmt.Errorf("unable to watch shards. %v", err) } } else { @@ -55,18 +54,18 @@ func subscribeBlockHeaders(c client.Client) error { // collation for the available shards in the SMC. The function calls // getEligibleNotary from the SMC and notary a collation if // conditions are met. -func checkSMCForNotary(c client.Client, head *types.Header) error { +func checkSMCForNotary(n sharding.Node, head *types.Header) error { log.Info("Checking if we are an eligible collation notary for a shard...") for s := int64(0); s < sharding.ShardCount; s++ { // Checks if we are an eligible notary according to the SMC. - addr, err := c.SMCCaller().GetNotaryInCommittee(&bind.CallOpts{}, big.NewInt(s)) + addr, err := n.SMCCaller().GetNotaryInCommittee(&bind.CallOpts{}, big.NewInt(s)) if err != nil { return err } // If output is non-empty and the addr == coinbase. - if addr == c.Account().Address { + if addr == n.Account().Address { log.Info(fmt.Sprintf("Selected as notary on shard: %d", s)) err := submitCollation(s) if err != nil { @@ -74,7 +73,7 @@ func checkSMCForNotary(c client.Client, head *types.Header) error { } // If the account is selected as notary, submit collation. - if addr == c.Account().Address { + if addr == n.Account().Address { log.Info(fmt.Sprintf("Selected as notary on shard: %d", s)) err := submitCollation(s) if err != nil { @@ -87,18 +86,18 @@ func checkSMCForNotary(c client.Client, head *types.Header) error { return nil } -// isAccountInNotaryPool checks if the client is in the notary pool because +// isAccountInNotaryPool checks if the user is in the notary pool because // we can't guarantee our tx for deposit will be in the next block header we receive. // The function calls IsNotaryDeposited from the SMC and returns true if -// the client is in the notary pool. -func isAccountInNotaryPool(c client.Client) (bool, error) { - account := c.Account() +// the user is in the notary pool. +func isAccountInNotaryPool(n sharding.Node) (bool, error) { + account := n.Account() // Checks if our deposit has gone through according to the SMC. - n, err := c.SMCCaller().NotaryRegistry(&bind.CallOpts{}, account.Address) - if !n.Deposited && err != nil { + nreg, err := n.SMCCaller().NotaryRegistry(&bind.CallOpts{}, account.Address) + if !nreg.Deposited && err != nil { log.Warn(fmt.Sprintf("Account %s not in notary pool.", account.Address.String())) } - return n.Deposited, err + return nreg.Deposited, err } // submitCollation interacts with the SMC directly to add a collation header. @@ -134,22 +133,22 @@ func submitCollation(shardID int64) error { // joinNotaryPool checks if the deposit flag is true and the account is a // notary in the SMC. If the account is not in the set, it will deposit ETH // into contract. -func joinNotaryPool(c client.Client) error { - if !c.DepositFlagSet() { +func joinNotaryPool(n sharding.Node) error { + if !n.DepositFlagSet() { return errors.New("joinNotaryPool called when deposit flag was not set") } - if b, err := isAccountInNotaryPool(c); b || err != nil { + if b, err := isAccountInNotaryPool(n); b || err != nil { return err } log.Info("Joining notary pool") - txOps, err := c.CreateTXOpts(sharding.NotaryDeposit) + txOps, err := n.CreateTXOpts(sharding.NotaryDeposit) if err != nil { return fmt.Errorf("unable to intiate the deposit transaction: %v", err) } - tx, err := c.SMCTransactor().RegisterNotary(txOps) + tx, err := n.SMCTransactor().RegisterNotary(txOps) if err != nil { return fmt.Errorf("unable to deposit eth and become a notary: %v", err) } diff --git a/sharding/notary/notary_client.go b/sharding/notary/notary_client.go deleted file mode 100644 index 029909b220f..00000000000 --- a/sharding/notary/notary_client.go +++ /dev/null @@ -1,42 +0,0 @@ -package notary - -import ( - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/sharding/client" - cli "gopkg.in/urfave/cli.v1" -) - -// Notary runnable client. -type Notary interface { - // Start the main routine for a notary. - Start() error -} - -type notary struct { - client client.Client -} - -// NewNotary creates a new notary instance. -func NewNotary(ctx *cli.Context) Notary { - return ¬ary{ - client: client.NewClient(ctx), - } -} - -// Start the main routine for a notary. -func (c *notary) Start() error { - log.Info("Starting notary client") - err := c.client.Start() - if err != nil { - return err - } - defer c.client.Close() - - if c.client.DepositFlagSet() { - if err := joinNotaryPool(c.client); err != nil { - return err - } - } - - return subscribeBlockHeaders(c.client) -} diff --git a/sharding/notary/service.go b/sharding/notary/service.go new file mode 100644 index 00000000000..f2ebbd5d60c --- /dev/null +++ b/sharding/notary/service.go @@ -0,0 +1,38 @@ +package notary + +import ( + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/sharding" + cli "gopkg.in/urfave/cli.v1" +) + +// Notary holds functionality required to run a collation notary +// in a sharded system. Must satisfy the Service interface defined in +// sharding/service.go. +type Notary struct { + node sharding.Node +} + +// NewNotary creates a new notary instance. +func NewNotary(ctx *cli.Context, node sharding.Node) *Notary { + return &Notary{node} +} + +// Start the main routine for a notary. +func (n *Notary) Start() error { + log.Info("Starting notary client") + // err := c.client.Start() + // if err != nil { + // return err + // } + // defer c.client.Close() + + // if c.client.DepositFlagSet() { + // if err := joinNotaryPool(c.client); err != nil { + // return err + // } + // } + + // return subscribeBlockHeaders(c.client) + return nil +} diff --git a/sharding/notary/notary_test.go b/sharding/notary/service_test.go similarity index 81% rename from sharding/notary/notary_test.go rename to sharding/notary/service_test.go index 21fa7d6fd6b..4c08f2ad412 100644 --- a/sharding/notary/notary_test.go +++ b/sharding/notary/service_test.go @@ -21,45 +21,46 @@ var ( ) // Mock client for testing notary. Should this go into sharding/client/testing? -type mockClient struct { +type mockNode struct { smc *contracts.SMC t *testing.T DepositFlag bool } -func (m *mockClient) Account() *accounts.Account { +func (m *mockNode) Account() *accounts.Account { return &accounts.Account{Address: addr} } -func (m *mockClient) SMCCaller() *contracts.SMCCaller { +func (m *mockNode) SMCCaller() *contracts.SMCCaller { return &m.smc.SMCCaller } -func (m *mockClient) ChainReader() ethereum.ChainReader { +func (m *mockNode) ChainReader() ethereum.ChainReader { m.t.Fatal("ChainReader not implemented") return nil } -func (m *mockClient) SMCTransactor() *contracts.SMCTransactor { +func (m *mockNode) SMCTransactor() *contracts.SMCTransactor { return &m.smc.SMCTransactor } -func (m *mockClient) CreateTXOpts(value *big.Int) (*bind.TransactOpts, error) { +func (m *mockNode) CreateTXOpts(value *big.Int) (*bind.TransactOpts, error) { txOpts := transactOpts() txOpts.Value = value return txOpts, nil } -func (m *mockClient) DepositFlagSet() bool { +func (m *mockNode) DepositFlagSet() bool { return m.DepositFlag } // Unused mockClient methods. -func (m *mockClient) Start() error { +func (m *mockNode) Start() error { m.t.Fatal("Start called") return nil } -func (m *mockClient) Close() { + +func (m *mockNode) Close() { m.t.Fatal("Close called") } @@ -69,6 +70,7 @@ func (m *mockClient) Close() { func transactOpts() *bind.TransactOpts { return bind.NewKeyedTransactor(key) } + func setup() (*backends.SimulatedBackend, *contracts.SMC) { backend := backends.NewSimulatedBackend(core.GenesisAlloc{addr: {Balance: accountBalance1001Eth}}) _, _, smc, _ := contracts.DeploySMC(transactOpts(), backend) @@ -78,10 +80,10 @@ func setup() (*backends.SimulatedBackend, *contracts.SMC) { func TestIsAccountInNotaryPool(t *testing.T) { backend, smc := setup() - client := &mockClient{smc: smc, t: t} + node := &mockNode{smc: smc, t: t} // address should not be in pool initially. - b, err := isAccountInNotaryPool(client) + b, err := isAccountInNotaryPool(node) if err != nil { t.Fatal(err) } @@ -96,7 +98,7 @@ func TestIsAccountInNotaryPool(t *testing.T) { t.Fatalf("Failed to deposit: %v", err) } backend.Commit() - b, err = isAccountInNotaryPool(client) + b, err = isAccountInNotaryPool(node) if err != nil { t.Fatal(err) } @@ -107,7 +109,7 @@ func TestIsAccountInNotaryPool(t *testing.T) { func TestJoinNotaryPool(t *testing.T) { backend, smc := setup() - client := &mockClient{smc: smc, t: t, DepositFlag: false} + node := &mockNode{smc: smc, t: t, DepositFlag: false} // There should be no notary initially. numNotaries, err := smc.NotaryPoolLength(&bind.CallOpts{}) if err != nil { @@ -117,14 +119,14 @@ func TestJoinNotaryPool(t *testing.T) { t.Fatalf("Unexpected number of notaries. Got %d, wanted 0.", numNotaries) } - client.DepositFlag = false - err = joinNotaryPool(client) + node.DepositFlag = false + err = joinNotaryPool(node) if err == nil { t.Error("Joined notary pool while --deposit was not present") } - client.DepositFlag = true - err = joinNotaryPool(client) + node.DepositFlag = true + err = joinNotaryPool(node) if err != nil { t.Fatal(err) } @@ -140,7 +142,7 @@ func TestJoinNotaryPool(t *testing.T) { } // Trying to join while deposited should do nothing - err = joinNotaryPool(client) + err = joinNotaryPool(node) if err != nil { t.Error(err) } diff --git a/sharding/proposer/proposer.go b/sharding/proposer/service.go similarity index 75% rename from sharding/proposer/proposer.go rename to sharding/proposer/service.go index d7aa1dba44f..f74a5b26e5f 100644 --- a/sharding/proposer/proposer.go +++ b/sharding/proposer/service.go @@ -2,8 +2,7 @@ package proposer import ( "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/sharding/node" - "github.com/ethereum/go-ethereum/swarm/api/client" + "github.com/ethereum/go-ethereum/sharding" cli "gopkg.in/urfave/cli.v1" ) @@ -11,14 +10,14 @@ import ( // in a sharded system. Must satisfy the Service interface defined in // sharding/service.go. type Proposer struct { - client client.Client + node sharding.Node } // NewProposer creates a struct instance. It is initialized and // registered as a service upon start of a sharding node. // Has access to the public methods of this node. -func NewProposer(ctx *cli.Context, node node.Node) *Proposer { - return &Proposer{} +func NewProposer(ctx *cli.Context, node sharding.Node) *Proposer { + return &Proposer{node} } // Start the main entry point for proposing collations. diff --git a/sharding/service.go b/sharding/types.go similarity index 52% rename from sharding/service.go rename to sharding/types.go index 389a6122240..721a34bd145 100644 --- a/sharding/service.go +++ b/sharding/types.go @@ -1,5 +1,14 @@ package sharding +import ( + "math/big" + + ethereum "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/sharding/contracts" +) + // Service defines items that can be registered into a sharding client. // // life-cycle management is delegated to the sharding client. The service is allowed to @@ -14,3 +23,15 @@ type Service interface { // are all terminated. Stop() error } + +// Node methods that must be implemented to run a sharding node. +type Node interface { + Start() error + Close() + CreateTXOpts(*big.Int) (*bind.TransactOpts, error) + ChainReader() ethereum.ChainReader + Account() *accounts.Account + SMCCaller() *contracts.SMCCaller + SMCTransactor() *contracts.SMCTransactor + DepositFlagSet() bool +} From b0a990e08bd34a6ac49f23773e01e5aaa78c1118 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 22 May 2018 07:47:35 -0500 Subject: [PATCH 06/16] sharding: fixed main entry point, linter errors --- cmd/geth/shardingcmd.go | 43 ++++++-- cmd/utils/flags.go | 4 +- sharding/{types.go => interfaces.go} | 21 +--- sharding/node/node.go | 147 +++++++++++++++------------ sharding/node/utils.go | 2 +- sharding/notary/notary.go | 9 +- sharding/notary/service.go | 28 ++--- sharding/notary/service_test.go | 9 ++ sharding/proposer/service.go | 19 ++-- 9 files changed, 161 insertions(+), 121 deletions(-) rename sharding/{types.go => interfaces.go} (53%) diff --git a/cmd/geth/shardingcmd.go b/cmd/geth/shardingcmd.go index 7358adc1f65..84d8f313835 100644 --- a/cmd/geth/shardingcmd.go +++ b/cmd/geth/shardingcmd.go @@ -1,8 +1,13 @@ package main import ( + "fmt" + "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/sharding/client" + "github.com/ethereum/go-ethereum/sharding" + "github.com/ethereum/go-ethereum/sharding/node" + "github.com/ethereum/go-ethereum/sharding/notary" + "github.com/ethereum/go-ethereum/sharding/proposer" cli "gopkg.in/urfave/cli.v1" ) @@ -12,7 +17,7 @@ var ( Name: "sharding", Usage: "Start a sharding-enabled node", ArgsUsage: "[endpoint]", - Flags: []cli.Flag{utils.DataDirFlag, utils.PasswordFileFlag, utils.NetworkIdFlag, utils.IPCPathFlag, utils.DepositFlag}, + Flags: []cli.Flag{utils.ProtocolFlag, utils.DataDirFlag, utils.PasswordFileFlag, utils.NetworkIdFlag, utils.IPCPathFlag, utils.DepositFlag}, Category: "SHARDING COMMANDS", Description: ` Launches a sharding client that submits collations to a Sharding Manager Contract, handles notary and proposer services, and manages shardp2p connections. This feature is a work in progress. @@ -22,16 +27,32 @@ Launches a sharding client that submits collations to a Sharding Manager Contrac func shardingClient(ctx *cli.Context) error { // configures a sharding-enabled node using the cli's context. - shardingNode := client.NewClient(ctx) + shardingNode := node.NewNode(ctx) + registerShardingServices(shardingNode) + defer shardingNode.Close() return shardingNode.Start() } -// func notaryClient(ctx *cli.Context) error { -// c := notary.NewNotary(ctx) -// return c.Start() -// } +// registerShardingServices sets up either a notary or proposer +// sharding service dependent on the ClientType cli flag. We should be defining +// the services we want to register here, as this is the geth command entry point +// for sharding. +func registerShardingServices(n node.Node) error { + protocolFlag := n.Context().GlobalString(utils.ProtocolFlag.Name) + + err := n.Register(func(ctx *cli.Context) (sharding.Service, error) { + // TODO(terenc3t): handle case when we just want to start an observer node. + if protocolFlag == "notary" { + return notary.NewNotary(n.Context(), n) + } + return proposer.NewProposer(n.Context(), n) + }) -// func proposerClient(ctx *cli.Context) error { -// p := proposer.NewProposer(ctx) -// return p.Start() -// } + if err != nil { + return fmt.Errorf("failed to register the main sharding services: %v", err) + } + + // TODO(prestonvanloon) registers the shardp2p service. + // we can do n.Register and initialize a shardp2p.NewServer() or something like that. + return nil +} diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index b4a3897ba05..5a3d863ebfc 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -539,8 +539,8 @@ var ( Name: "deposit", Usage: "To become a notary with your sharding client, " + new(big.Int).Div(sharding.NotaryDeposit, new(big.Int).Exp(big.NewInt(10), big.NewInt(18), nil)).String() + " ETH will be deposited into SMC", } - ClientType = cli.StringFlag{ - Name: "client-type", + ProtocolFlag = cli.StringFlag{ + Name: "protocol", Usage: "notary | proposer", } ) diff --git a/sharding/types.go b/sharding/interfaces.go similarity index 53% rename from sharding/types.go rename to sharding/interfaces.go index 721a34bd145..a515fc35984 100644 --- a/sharding/types.go +++ b/sharding/interfaces.go @@ -1,12 +1,7 @@ package sharding import ( - "math/big" - - ethereum "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/accounts" - "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/sharding/contracts" + cli "gopkg.in/urfave/cli.v1" ) // Service defines items that can be registered into a sharding client. @@ -24,14 +19,6 @@ type Service interface { Stop() error } -// Node methods that must be implemented to run a sharding node. -type Node interface { - Start() error - Close() - CreateTXOpts(*big.Int) (*bind.TransactOpts, error) - ChainReader() ethereum.ChainReader - Account() *accounts.Account - SMCCaller() *contracts.SMCCaller - SMCTransactor() *contracts.SMCTransactor - DepositFlagSet() bool -} +// ServiceConstructor defines the callback passed in when registering a service +// to a sharding node. +type ServiceConstructor func(ctx *cli.Context) (Service, error) diff --git a/sharding/node/node.go b/sharding/node/node.go index 0aacaf43c88..d9e2188644a 100644 --- a/sharding/node/node.go +++ b/sharding/node/node.go @@ -1,4 +1,4 @@ -// Package sharding/node provides an interface for interacting with a running ethereum full node. +// Package node provides an interface for interacting with a running ethereum full node. // As part of the initial phases of sharding, actors will need access to the sharding management // contract on the main PoW chain. package node @@ -10,6 +10,7 @@ import ( "fmt" "math/big" "os" + "sync" ethereum "github.com/ethereum/go-ethereum" @@ -22,9 +23,8 @@ import ( "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/sharding" "github.com/ethereum/go-ethereum/sharding/contracts" - "github.com/ethereum/go-ethereum/sharding/notary" - "github.com/ethereum/go-ethereum/sharding/proposer" cli "gopkg.in/urfave/cli.v1" ) @@ -32,20 +32,36 @@ const ( clientIdentifier = "geth" // Used to determine the ipc name. ) +// Node methods that must be implemented to run a sharding node. +type Node interface { + Start() error + Close() + Context() *cli.Context + Register(sharding.ServiceConstructor) error + CreateTXOpts(*big.Int) (*bind.TransactOpts, error) + ChainReader() ethereum.ChainReader + Account() *accounts.Account + SMCCaller() *contracts.SMCCaller + SMCTransactor() *contracts.SMCTransactor + DepositFlagSet() bool +} + // General node for a sharding-enabled system. // Communicates to Geth node via JSON RPC. type shardingNode struct { - endpoint string // Endpoint to JSON RPC. - client *ethclient.Client // Ethereum RPC client. - keystore *keystore.KeyStore // Keystore containing the single signer. - ctx *cli.Context // Command line context. - smc *contracts.SMC // The deployed sharding management contract. - rpcClient *rpc.Client // The RPC client connection to the main geth node. + endpoint string // Endpoint to JSON RPC. + client *ethclient.Client // Ethereum RPC client. + keystore *keystore.KeyStore // Keystore containing the single signer. + ctx *cli.Context // Command line context. + smc *contracts.SMC // The deployed sharding management contract. + rpcClient *rpc.Client // The RPC client connection to the main geth node. + lock sync.RWMutex // Mutex lock for concurrency management. + serviceFuncs []sharding.ServiceConstructor // Stores an array of service callbacks to start upon running. } -// NewCNode setups the sharding config, registers the services required +// NewNode setups the sharding config, registers the services required // by the sharded system. -func NewNode(ctx *cli.Context) Client { +func NewNode(ctx *cli.Context) Node { c := &shardingNode{ctx: ctx} // Sets up all configuration options based on cli flags. @@ -53,14 +69,53 @@ func NewNode(ctx *cli.Context) Client { panic(err) // TODO(rauljordan): handle this } - // Registers all required services the sharding node will run upon start. - // These include shardp2p servers, notary/proposer event loops, and more. - if err := c.registerShardingServices(); err != nil { - panic(err) // TODO(rauljordan): handle this. - } return c } +// Start is the main entrypoint of a sharding node. It starts off every service +// attached to it. +func (n *shardingNode) Start() error { + // Sets up a connection to a Geth node via RPC. + rpcClient, err := dialRPC(n.endpoint) + if err != nil { + return fmt.Errorf("cannot start rpc client. %v", err) + } + n.rpcClient = rpcClient + n.client = ethclient.NewClient(rpcClient) + + // Check account existence and unlock account before starting. + accounts := n.keystore.Accounts() + if len(accounts) == 0 { + return fmt.Errorf("no accounts found") + } + + if err := n.unlockAccount(accounts[0]); err != nil { + return fmt.Errorf("cannot unlock account. %v", err) + } + + // Initializes bindings to SMC. + smc, err := initSMC(n) + if err != nil { + return err + } + n.smc = smc + + // Starts every service attached to the sharding node. + for _, serviceFunc := range n.serviceFuncs { + // Initializes each service by passing in the node's cli context. + service, err := serviceFunc(n.ctx) + if err != nil { + return err + } + if err := service.Start(); err != nil { + // Handles the stopping of a service on error. + service.Stop() + return err + } + } + return nil +} + // configShardingNode uses cli flags to configure the data // directory, ipc endpoints, keystores, and more. func (n *shardingNode) configShardingNode() error { @@ -94,62 +149,15 @@ func (n *shardingNode) configShardingNode() error { return nil } -// registerShardingServices sets up either a notary or proposer -// sharding service dependent on the ClientType cli flag -func (n *shardingNode) registerShardingServices() error { - clientType := n.ctx.GlobalString(utils.ClientType.Name) - - err := n.Register(func(ctx *node.ServiceContext) (node.Service, error) { - // TODO(terenc3t): handle case when we just want to start an observer node. - if clientType == "notary" { - return notary.NewNotary(n.ctx) - } - return proposer.NewProposer(n.ctx) - }) - - if err != nil { - return fmt.Errorf("failed to register the main sharding services: %v", err) - } - // TODO: registers the shardp2p service. - - return nil -} - // Register appends a struct to the sharding node's services that // satisfies the Service interface containing lifecycle handlers. Notary, Proposer, // and ShardP2P are examples of services. The rationale behind this is that the // sharding node should know very little about the functioning of its underlying // services as they should be extensible. -func (n *shardingNode) Register(a interface{}) error { - return nil -} - -// Start is the main entrypoint of a sharding node. It starts off every service -// attached to it. -func (n *shardingNode Start() error { - rpcClient, err := dialRPC(c.endpoint) - if err != nil { - return fmt.Errorf("cannot start rpc client. %v", err) - } - n.rpcClient = rpcClient - n.client = ethclient.NewClient(rpcClient) - - // Check account existence and unlock account before starting notary client. - accounts := n.keystore.Accounts() - if len(accounts) == 0 { - return fmt.Errorf("no accounts found") - } - - if err := n.unlockAccount(accounts[0]); err != nil { - return fmt.Errorf("cannot unlock account. %v", err) - } - - smc, err := initSMC(c) - if err != nil { - return err - } - n.smc = smc - +func (n *shardingNode) Register(service sharding.ServiceConstructor) error { + n.lock.Lock() + defer n.lock.Unlock() + n.serviceFuncs = append(n.serviceFuncs, service) return nil } @@ -182,6 +190,11 @@ func (n *shardingNode) Account() *accounts.Account { return &accounts[0] } +// Context returns the cli context. +func (n *shardingNode) Context() *cli.Context { + return n.ctx +} + // ChainReader for interacting with the chain. func (n *shardingNode) ChainReader() ethereum.ChainReader { return ethereum.ChainReader(n.client) diff --git a/sharding/node/utils.go b/sharding/node/utils.go index 61503259d90..3f6b7cdae9f 100644 --- a/sharding/node/utils.go +++ b/sharding/node/utils.go @@ -40,7 +40,7 @@ func initSMC(n *shardingNode) (*contracts.SMC, error) { return nil, fmt.Errorf("unable to intiate the transaction: %v", err) } - addr, tx, contract, err := contracts.DeploySMC(txOps, c.client) + addr, tx, contract, err := contracts.DeploySMC(txOps, n.client) if err != nil { return nil, fmt.Errorf("unable to deploy sharding manager contract: %v", err) } diff --git a/sharding/notary/notary.go b/sharding/notary/notary.go index a96de82afa0..0934222a039 100644 --- a/sharding/notary/notary.go +++ b/sharding/notary/notary.go @@ -11,6 +11,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/sharding" + "github.com/ethereum/go-ethereum/sharding/node" ) // SubscribeBlockHeaders checks incoming block headers and determines if @@ -18,7 +19,7 @@ import ( // from the running geth node and sorts them by descending order of gas price, // eliminates those that ask for too much gas, and routes them over // to the SMC to create a collation. -func subscribeBlockHeaders(n sharding.Node) error { +func subscribeBlockHeaders(n node.Node) error { headerChan := make(chan *types.Header, 16) _, err := n.ChainReader().SubscribeNewHead(context.Background(), headerChan) @@ -54,7 +55,7 @@ func subscribeBlockHeaders(n sharding.Node) error { // collation for the available shards in the SMC. The function calls // getEligibleNotary from the SMC and notary a collation if // conditions are met. -func checkSMCForNotary(n sharding.Node, head *types.Header) error { +func checkSMCForNotary(n node.Node, head *types.Header) error { log.Info("Checking if we are an eligible collation notary for a shard...") for s := int64(0); s < sharding.ShardCount; s++ { // Checks if we are an eligible notary according to the SMC. @@ -90,7 +91,7 @@ func checkSMCForNotary(n sharding.Node, head *types.Header) error { // we can't guarantee our tx for deposit will be in the next block header we receive. // The function calls IsNotaryDeposited from the SMC and returns true if // the user is in the notary pool. -func isAccountInNotaryPool(n sharding.Node) (bool, error) { +func isAccountInNotaryPool(n node.Node) (bool, error) { account := n.Account() // Checks if our deposit has gone through according to the SMC. nreg, err := n.SMCCaller().NotaryRegistry(&bind.CallOpts{}, account.Address) @@ -133,7 +134,7 @@ func submitCollation(shardID int64) error { // joinNotaryPool checks if the deposit flag is true and the account is a // notary in the SMC. If the account is not in the set, it will deposit ETH // into contract. -func joinNotaryPool(n sharding.Node) error { +func joinNotaryPool(n node.Node) error { if !n.DepositFlagSet() { return errors.New("joinNotaryPool called when deposit flag was not set") } diff --git a/sharding/notary/service.go b/sharding/notary/service.go index f2ebbd5d60c..698ac9d8b27 100644 --- a/sharding/notary/service.go +++ b/sharding/notary/service.go @@ -2,7 +2,7 @@ package notary import ( "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/sharding" + "github.com/ethereum/go-ethereum/sharding/node" cli "gopkg.in/urfave/cli.v1" ) @@ -10,29 +10,31 @@ import ( // in a sharded system. Must satisfy the Service interface defined in // sharding/service.go. type Notary struct { - node sharding.Node + node node.Node } // NewNotary creates a new notary instance. -func NewNotary(ctx *cli.Context, node sharding.Node) *Notary { - return &Notary{node} +func NewNotary(ctx *cli.Context, node node.Node) (*Notary, error) { + return &Notary{node}, nil } // Start the main routine for a notary. func (n *Notary) Start() error { - log.Info("Starting notary client") - // err := c.client.Start() - // if err != nil { - // return err - // } - // defer c.client.Close() + log.Info("Starting notary service") - // if c.client.DepositFlagSet() { - // if err := joinNotaryPool(c.client); err != nil { + // if n.node.DepositFlagSet() { + // if err := joinNotaryPool(n.node); err != nil { // return err // } // } - // return subscribeBlockHeaders(c.client) + // return subscribeBlockHeaders(n.node) + return nil +} + +// Stop the main loop for notarizing collations. +func (n *Notary) Stop() error { + log.Info("Stopping notary service") + // TODO: Propose collations. return nil } diff --git a/sharding/notary/service_test.go b/sharding/notary/service_test.go index 4c08f2ad412..52fb00e3788 100644 --- a/sharding/notary/service_test.go +++ b/sharding/notary/service_test.go @@ -12,6 +12,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/sharding" "github.com/ethereum/go-ethereum/sharding/contracts" + cli "gopkg.in/urfave/cli.v1" ) var ( @@ -40,6 +41,14 @@ func (m *mockNode) ChainReader() ethereum.ChainReader { return nil } +func (m *mockNode) Context() *cli.Context { + return nil +} + +func (m *mockNode) Register(s sharding.ServiceConstructor) error { + return nil +} + func (m *mockNode) SMCTransactor() *contracts.SMCTransactor { return &m.smc.SMCTransactor } diff --git a/sharding/proposer/service.go b/sharding/proposer/service.go index f74a5b26e5f..91719cd7c1c 100644 --- a/sharding/proposer/service.go +++ b/sharding/proposer/service.go @@ -2,7 +2,7 @@ package proposer import ( "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/sharding" + "github.com/ethereum/go-ethereum/sharding/node" cli "gopkg.in/urfave/cli.v1" ) @@ -10,19 +10,26 @@ import ( // in a sharded system. Must satisfy the Service interface defined in // sharding/service.go. type Proposer struct { - node sharding.Node + node node.Node } // NewProposer creates a struct instance. It is initialized and // registered as a service upon start of a sharding node. // Has access to the public methods of this node. -func NewProposer(ctx *cli.Context, node sharding.Node) *Proposer { - return &Proposer{node} +func NewProposer(ctx *cli.Context, node node.Node) (*Proposer, error) { + return &Proposer{node}, nil } -// Start the main entry point for proposing collations. +// Start the main loop for proposing collations. func (p *Proposer) Start() error { - log.Info("Starting proposer client") + log.Info("Starting proposer service") + // TODO: Propose collations. + return nil +} + +// Stop the main loop for proposing collations. +func (p *Proposer) Stop() error { + log.Info("Stopping proposer service") // TODO: Propose collations. return nil } From d6a365c9054fd7ba200a4d27cbc7db9aee961982 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 22 May 2018 07:56:56 -0500 Subject: [PATCH 07/16] sharding: fully functional start func in notary service --- sharding/notary/service.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/sharding/notary/service.go b/sharding/notary/service.go index 698ac9d8b27..8aff199346c 100644 --- a/sharding/notary/service.go +++ b/sharding/notary/service.go @@ -22,14 +22,16 @@ func NewNotary(ctx *cli.Context, node node.Node) (*Notary, error) { func (n *Notary) Start() error { log.Info("Starting notary service") - // if n.node.DepositFlagSet() { - // if err := joinNotaryPool(n.node); err != nil { - // return err - // } - // } + // TODO: handle this better through goroutines. Right now, these methods + // have their own nested channels and goroutines within them. We need + // to make this as flat as possible at the Notary layer. + if n.node.DepositFlagSet() { + if err := joinNotaryPool(n.node); err != nil { + return err + } + } - // return subscribeBlockHeaders(n.node) - return nil + return subscribeBlockHeaders(n.node) } // Stop the main loop for notarizing collations. From 710e118d6df769ade926c7bd13561cbe70195c65 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 22 May 2018 08:11:16 -0500 Subject: [PATCH 08/16] sharding: travis passes, all entrypoints work, notary protocol working --- cmd/geth/main.go | 1 + cmd/geth/shardingcmd.go | 1 + 2 files changed, 2 insertions(+) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 25239247d33..2518469c5c4 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -119,6 +119,7 @@ var ( utils.GpoPercentileFlag, utils.ExtraDataFlag, utils.DepositFlag, + utils.ProtocolFlag, configFileFlag, } diff --git a/cmd/geth/shardingcmd.go b/cmd/geth/shardingcmd.go index 84d8f313835..5c37e412f3b 100644 --- a/cmd/geth/shardingcmd.go +++ b/cmd/geth/shardingcmd.go @@ -30,6 +30,7 @@ func shardingClient(ctx *cli.Context) error { shardingNode := node.NewNode(ctx) registerShardingServices(shardingNode) defer shardingNode.Close() + // starts a connection to a geth node and kicks off every registered service. return shardingNode.Start() } From e4c86822de463778f50e2d17e710579e8215bb57 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 22 May 2018 11:33:46 -0400 Subject: [PATCH 09/16] sharding: update documentation to reflect new entry points --- README.md | 18 +++++++++--------- cmd/geth/shardingcmd.go | 2 +- cmd/utils/flags.go | 2 +- sharding/README.md | 20 ++++++++++---------- sharding/interfaces.go | 4 ++-- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 7b7605a2bdd..9bbbd8c1e2d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ![Travis Build](https://travis-ci.org/prysmaticlabs/geth-sharding.svg?branch=master) -This is the main repository for the sharding implementation of the go-ethereum client by [Prysmatic Labs](https://prysmaticlabs.com). For the original, go-ethereum project, refer to the following [link](https://github.com/ethereum/go-ethereum). +This is the main repository for the sharding implementation for the go-ethereum project by [Prysmatic Labs](https://prysmaticlabs.com). For the original, go-ethereum project, refer to the following [link](https://github.com/ethereum/go-ethereum). Before you begin, check out our [Sharding Reference Implementation Doc](https://github.com/prysmaticlabs/geth-sharding/blob/master/sharding/README.md). This doc serves as the single source of truth for our team, our milestones, and details on the different components of our architecture. @@ -17,7 +17,7 @@ Interested in contributing? Check out our [Contribution Guidelines](#contributio - [Running a Local Geth Node](#running-a-local-geth-node) - [Transaction Generator](#transaction-generator) - [Becoming a Notary](#becoming-a-notary) - - [Running a Collation Proposal Client](#running-a-collation-proposal-client) + - [Running a Collation Proposal Node](#running-a-collation-proposal-node) - [Testing](#testing) - [Contribution Guidelines](#contribution-guidelines) - [License](#license) @@ -107,26 +107,26 @@ Our system outlined below follows the [Minimal Sharding Protocol](https://ethres To deposit ETH and join as a notary in the Sharding Manager Contract, run the following command: ``` -geth sharding-notary --deposit --datadir /path/to/your/datadir --password /path/to/your/password.txt --networkid 12345 +geth sharding --protocol "notary" --deposit --datadir /path/to/your/datadir --password /path/to/your/password.txt --networkid 12345 ``` -This will extract 1000ETH from your account balance and insert you into the SMC's notaries. Then, the program will listen for incoming block headers and notify you when you have been selected as to vote on proposals for a certain shard in a given period. Once you are selected, your sharding client will download collation information to check for data availability on vote on proposals that have been submitted via the `addHeader` function on the SMC. +This will extract 1000ETH from your account balance and insert you into the SMC's notaries. Then, the program will listen for incoming block headers and notify you when you have been selected as to vote on proposals for a certain shard in a given period. Once you are selected, your sharding node will download collation information to check for data availability on vote on proposals that have been submitted via the `addHeader` function on the SMC. -Concurrently, you will need to run another client that is tasked with processing transactions into collations and submitting them to the SMC via the `addHeader` function. +Concurrently, you will need to run another node that is tasked with processing transactions into collations and submitting them to the SMC via the `addHeader` function. -## Running a Collation Proposal Client +## Running a Collation Proposal Node ``` -geth sharding-proposer --datadir /path/to/your/datadir --password /path/to/your/password.txt --networkid 12345 +geth sharding --protocol "proposer" --datadir /path/to/your/datadir --password /path/to/your/password.txt --networkid 12345 ``` -This client is tasked with processing pending transactions into blobs within collations by serializing data into collation bodies. It is responsible for submitting proposals (collation headers) to the SMC via the `addHeader` function. +This node is tasked with processing pending transactions into blobs within collations by serializing data into collation bodies. It is responsible for submitting proposals (collation headers) to the SMC via the `addHeader` function. # Making Changes ## Rebuilding the Sharding Manager Contract Bindings -The Sharding Manager Contract is built in Solidity and deployed to the geth node upon launch of the client if it does not exist in the network at a specified address. If there are any changes to the SMC's code, the Golang bindigs must be rebuilt with the following command. +The Sharding Manager Contract is built in Solidity and deployed to a running geth node upon launch of the sharding node if it does not exist in the network at a specified address. If there are any changes to the SMC's code, the Golang bindigs must be rebuilt with the following command. go generate github.com/ethereum/go-ethereum/sharding # OR diff --git a/cmd/geth/shardingcmd.go b/cmd/geth/shardingcmd.go index 5c37e412f3b..0030a48e238 100644 --- a/cmd/geth/shardingcmd.go +++ b/cmd/geth/shardingcmd.go @@ -20,7 +20,7 @@ var ( Flags: []cli.Flag{utils.ProtocolFlag, utils.DataDirFlag, utils.PasswordFileFlag, utils.NetworkIdFlag, utils.IPCPathFlag, utils.DepositFlag}, Category: "SHARDING COMMANDS", Description: ` -Launches a sharding client that submits collations to a Sharding Manager Contract, handles notary and proposer services, and manages shardp2p connections. This feature is a work in progress. +Launches a sharding node that manages services related to submitting collations to a Sharding Manager Contract, notary and proposer services, and shardp2p connections. This feature is a work in progress. `, } ) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 5a3d863ebfc..5bbf58b5235 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -537,7 +537,7 @@ var ( // Sharding Settings DepositFlag = cli.BoolFlag{ Name: "deposit", - Usage: "To become a notary with your sharding client, " + new(big.Int).Div(sharding.NotaryDeposit, new(big.Int).Exp(big.NewInt(10), big.NewInt(18), nil)).String() + " ETH will be deposited into SMC", + Usage: "To become a notary in a sharding node, " + new(big.Int).Div(sharding.NotaryDeposit, new(big.Int).Exp(big.NewInt(10), big.NewInt(18), nil)).String() + " ETH will be deposited into SMC", } ProtocolFlag = cli.StringFlag{ Name: "protocol", diff --git a/sharding/README.md b/sharding/README.md index 098a0757daa..3e9c41153ff 100644 --- a/sharding/README.md +++ b/sharding/README.md @@ -119,9 +119,9 @@ To concretize these phases, we will be releasing our implementation of sharding Our current work is focused on creating a localized version of phase 1, quadratic sharding that would include the following: -- A minimal, **sharding client** system that will interact with a **Sharding Manager Contract** on a locally running geth node +- A minimal, **sharding node** system that will interact with a **Sharding Manager Contract** on a locally running geth node - Ability to deposit ETH into the SMC through the command line and to be selected as a notary by the local **SMC** in addition to the ability to withdraw the ETH staked -- A **proposer client** that listens for pending tx’s, creates collations, and submits them to the SMC +- A **proposer protocol** that listens for pending tx’s, creates collations, and submits them to the SMC - Ability to inspect the shard states and visualize the working system locally @@ -147,7 +147,7 @@ ETA: To Be determined # Go-Ethereum Sharding Alpha Implementation -Prysmatic Labs will begin by focusing its implementation entirely on the **Ruby Release** from our roadmap. We plan on being as pragmatic as possible to create something that can be locally run by any developer as soon as possible. Our initial deliverable will center around a command line tool that will serve as an entrypoint into a sharding client that allows staking to become a notary, a proposer client that allows for the creation of collation proposals, shard state local storage, and on-chain voting of collation headers via the Sharding Manager Contract. +Prysmatic Labs will begin by focusing its implementation entirely on the **Ruby Release** from our roadmap. We plan on being as pragmatic as possible to create something that can be locally run by any developer as soon as possible. Our initial deliverable will center around a command line tool that will serve as an entrypoint into a sharding node that allows staking to become a notary, proposer, manages shard state local storage, and does on-chain voting of collation headers via the Sharding Manager Contract. Here is a full reference spec explaining how our initial system will function: @@ -157,19 +157,19 @@ Our implementation revolves around 5 core components: - A **locally-running geth node** that spins up an instance of the Ethereum blockchain and mines on the Proof of Work chain - A **Sharding Manager Contract (SMC)** that is deployed onto this blockchain instance -- A **sharding client** that connects to the running geth node through JSON-RPC, provides bindings to the SMC -- A **notary client** that allows users to stake ETH into the SMC and be selected as a notary in a certain period on a shard -- A **proposer client** that is tasked with processing pending tx's into collations that are then submitted to the SMC. In phase 1, proposers _do not_ execute state, but rather just serialize pending tx data into possibly valid/invalid data blobs. +- A **sharding node** that connects to the running geth node through JSON-RPC, provides bindings to the SMC +- A **notary protocol** that allows users to stake ETH into the SMC and be selected as a notary in a certain period on a shard +- A **proposer protocol** that is tasked with processing pending tx's into collations that are then submitted to the SMC. In phase 1, proposers _do not_ execute state, but rather just serialize pending tx data into possibly valid/invalid data blobs. Our initial implementation will function through simple command line arguments that will allow a user running the local geth node to deposit ETH into the SMC and join as a notary that is randomly assigned to a shard in a certain period. A basic, end-to-end example of the system is as follows: -1. _**User starts a notary client and deposits 1000ETH into the SMC:**_ the sharding client connects to a locally running geth node and asks the user to confirm a deposit from his/her personal account. +1. _**User starts a sharding node and deposits 1000ETH into the SMC:**_ the sharding node connects to a locally running geth node and asks the user to confirm a deposit from his/her personal account. 2. _**Client connects & listens to incoming headers from the geth node and assigns user as notary on a shard per period:**_ The notary is selected in CURRENT_PERIOD + LOOKEAD_PERIOD (which is around a 5 minutes notice) and must download data for collation headers submitted in that time period. -3. _**Concurrently, a proposer client processes pending transaction data into blobs:**_ the proposer client will create collation bodies and submit their headers to the SMC. In Phase 1, it is important to note that we will _not_ have any state execution. Proposers will just serialize pending tx into fixed collation body sizes without executing them for state transition validity. +3. _**Concurrently, a proposer protocol processes pending transaction data into blobs:**_ the proposer client will create collation bodies and submit their headers to the SMC. In Phase 1, it is important to note that we will _not_ have any state execution. Proposers will just serialize pending tx into fixed collation body sizes without executing them for state transition validity. 5. _**The set of notaries vote on collation headers as canonical unitl the period ends:**_ the headers that received >= 2/3 votes are accepted as canonical. @@ -183,7 +183,7 @@ Our Ruby Release requires users to start a local geth node running a localized, geth sharding-notary --deposit --datadir /path/to/your/datadir --password /path/to/your/password.txt --networkid 12345 -This will extract 1000ETH from the user's account balance and insert him/her into the SMC's notaries. Then, the program will listen for incoming block headers and notify the user when he/she has been selected as to vote on collations for a certain shard in a given period. Once you are selected, the sharding client will download collation information to check for data availability on vote on proposals that have been submitted via the `addHeader` function on the SMC. +This will extract 1000ETH from the user's account balance and insert him/her into the SMC's notaries. Then, the program will listen for incoming block headers and notify the user when he/she has been selected as to vote on collations for a certain shard in a given period. Once you are selected, the sharding node will download collation information to check for data availability on vote on proposals that have been submitted via the `addHeader` function on the SMC. Users can also run a proposer client that is tasked with processing transactions into collations and submitting them to the SMC via the `addHeader` function. @@ -191,7 +191,7 @@ Users can also run a proposer client that is tasked with processing transactions This client is tasked with processing pending transactions into blobs within collations by serializing data into collation bodies. It is responsible for submitting proposals (collation headers) to the SMC via the `addHeader` function. -The sharding client begins to work by its main loop, which involves the following steps: +The sharding node begins to work by its main loop, which involves the following steps: 1. _**Subscribe to incoming block headers:**_ the client will begin by issuing a subscription over JSON-RPC for block headers from the running geth node. diff --git a/sharding/interfaces.go b/sharding/interfaces.go index a515fc35984..a75a04ca625 100644 --- a/sharding/interfaces.go +++ b/sharding/interfaces.go @@ -4,9 +4,9 @@ import ( cli "gopkg.in/urfave/cli.v1" ) -// Service defines items that can be registered into a sharding client. +// Service defines items that can be registered into a sharding node's serviceFuncs. // -// life-cycle management is delegated to the sharding client. The service is allowed to +// life-cycle management is delegated to the sharding node. The service is allowed to // initialize itself upon creation, but no goroutines should be spun up outside of the // Start method. type Service interface { From f7e67ea934c09f83a73f5d08212ce174914f176e Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 22 May 2018 11:37:06 -0400 Subject: [PATCH 10/16] sharding: elim unnecessary comment --- sharding/proposer/service.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sharding/proposer/service.go b/sharding/proposer/service.go index 91719cd7c1c..a1dccd2eda0 100644 --- a/sharding/proposer/service.go +++ b/sharding/proposer/service.go @@ -30,6 +30,5 @@ func (p *Proposer) Start() error { // Stop the main loop for proposing collations. func (p *Proposer) Stop() error { log.Info("Stopping proposer service") - // TODO: Propose collations. return nil } From 8193dce8776567cc70a26f80deab422bcaa65dbe Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 22 May 2018 12:42:49 -0400 Subject: [PATCH 11/16] sharding: address review comments, add observer package --- README.md | 10 ++++++++-- cmd/geth/main.go | 2 +- cmd/geth/shardingcmd.go | 20 ++++++++++++-------- cmd/utils/flags.go | 7 +++---- sharding/README.md | 10 +++++----- sharding/notary/service.go | 1 - sharding/observer/service.go | 33 +++++++++++++++++++++++++++++++++ 7 files changed, 62 insertions(+), 21 deletions(-) create mode 100644 sharding/observer/service.go diff --git a/README.md b/README.md index 9bbbd8c1e2d..1ccb22e3b9e 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ Our system outlined below follows the [Minimal Sharding Protocol](https://ethres To deposit ETH and join as a notary in the Sharding Manager Contract, run the following command: ``` -geth sharding --protocol "notary" --deposit --datadir /path/to/your/datadir --password /path/to/your/password.txt --networkid 12345 +geth sharding --actor "notary" --deposit --datadir /path/to/your/datadir --password /path/to/your/password.txt --networkid 12345 ``` This will extract 1000ETH from your account balance and insert you into the SMC's notaries. Then, the program will listen for incoming block headers and notify you when you have been selected as to vote on proposals for a certain shard in a given period. Once you are selected, your sharding node will download collation information to check for data availability on vote on proposals that have been submitted via the `addHeader` function on the SMC. @@ -117,11 +117,17 @@ Concurrently, you will need to run another node that is tasked with processing t ## Running a Collation Proposal Node ``` -geth sharding --protocol "proposer" --datadir /path/to/your/datadir --password /path/to/your/password.txt --networkid 12345 +geth sharding --actor "proposer" --datadir /path/to/your/datadir --password /path/to/your/password.txt --networkid 12345 ``` This node is tasked with processing pending transactions into blobs within collations by serializing data into collation bodies. It is responsible for submitting proposals (collation headers) to the SMC via the `addHeader` function. +## Running an Observer Node + + geth sharding --datadir /path/to/your/datadir --password /path/to/your/password.txt --networkid 12345 + +Omitting the `--actor` flag will launch a simple observer service attached to the sharding client that is able to listen to changes happening throughout the sharded Ethereum network. + # Making Changes ## Rebuilding the Sharding Manager Contract Bindings diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 2518469c5c4..1e4e6aebba8 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -119,7 +119,7 @@ var ( utils.GpoPercentileFlag, utils.ExtraDataFlag, utils.DepositFlag, - utils.ProtocolFlag, + utils.ActorFlag, configFileFlag, } diff --git a/cmd/geth/shardingcmd.go b/cmd/geth/shardingcmd.go index 0030a48e238..4e8cff57b10 100644 --- a/cmd/geth/shardingcmd.go +++ b/cmd/geth/shardingcmd.go @@ -4,20 +4,20 @@ import ( "fmt" "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/sharding" "github.com/ethereum/go-ethereum/sharding/node" "github.com/ethereum/go-ethereum/sharding/notary" + "github.com/ethereum/go-ethereum/sharding/observer" "github.com/ethereum/go-ethereum/sharding/proposer" cli "gopkg.in/urfave/cli.v1" ) var ( shardingCommand = cli.Command{ - Action: utils.MigrateFlags(shardingClient), + Action: utils.MigrateFlags(sharding), Name: "sharding", Usage: "Start a sharding-enabled node", ArgsUsage: "[endpoint]", - Flags: []cli.Flag{utils.ProtocolFlag, utils.DataDirFlag, utils.PasswordFileFlag, utils.NetworkIdFlag, utils.IPCPathFlag, utils.DepositFlag}, + Flags: []cli.Flag{utils.ActorFlag, utils.DataDirFlag, utils.PasswordFileFlag, utils.NetworkIdFlag, utils.IPCPathFlag, utils.DepositFlag}, Category: "SHARDING COMMANDS", Description: ` Launches a sharding node that manages services related to submitting collations to a Sharding Manager Contract, notary and proposer services, and shardp2p connections. This feature is a work in progress. @@ -25,7 +25,10 @@ Launches a sharding node that manages services related to submitting collations } ) -func shardingClient(ctx *cli.Context) error { +// sharding is the main cmd line entry point for starting a sharding-enabled node. +// A sharding node launches a suite of services including notary services, +// proposer services, and a shardp2p protocol. +func sharding(ctx *cli.Context) error { // configures a sharding-enabled node using the cli's context. shardingNode := node.NewNode(ctx) registerShardingServices(shardingNode) @@ -39,14 +42,15 @@ func shardingClient(ctx *cli.Context) error { // the services we want to register here, as this is the geth command entry point // for sharding. func registerShardingServices(n node.Node) error { - protocolFlag := n.Context().GlobalString(utils.ProtocolFlag.Name) + actorFlag := n.Context().GlobalString(utils.ActorFlag.Name) err := n.Register(func(ctx *cli.Context) (sharding.Service, error) { - // TODO(terenc3t): handle case when we just want to start an observer node. - if protocolFlag == "notary" { + if actorFlag == "notary" { return notary.NewNotary(n.Context(), n) + } else if actorFlag == "proposer" { + return proposer.NewProposer(n.Context(), n) } - return proposer.NewProposer(n.Context(), n) + return observer.NewObserver(n.Context(), n) }) if err != nil { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 5bbf58b5235..087ee74ba98 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -533,15 +533,14 @@ var ( Usage: "Minimum POW accepted", Value: whisper.DefaultMinimumPoW, } - // Sharding Settings DepositFlag = cli.BoolFlag{ Name: "deposit", Usage: "To become a notary in a sharding node, " + new(big.Int).Div(sharding.NotaryDeposit, new(big.Int).Exp(big.NewInt(10), big.NewInt(18), nil)).String() + " ETH will be deposited into SMC", } - ProtocolFlag = cli.StringFlag{ - Name: "protocol", - Usage: "notary | proposer", + ActorFlag = cli.StringFlag{ + Name: "actor", + Usage: `use the --actor "notary" or --actor "proposer" to start a notary or proposer service in the sharding node. If omitted, the sharding node registers an Observer service that simply observes the activity in the sharded network`, } ) diff --git a/sharding/README.md b/sharding/README.md index 3e9c41153ff..c0abdae5054 100644 --- a/sharding/README.md +++ b/sharding/README.md @@ -121,7 +121,7 @@ Our current work is focused on creating a localized version of phase 1, quadrati - A minimal, **sharding node** system that will interact with a **Sharding Manager Contract** on a locally running geth node - Ability to deposit ETH into the SMC through the command line and to be selected as a notary by the local **SMC** in addition to the ability to withdraw the ETH staked -- A **proposer protocol** that listens for pending tx’s, creates collations, and submits them to the SMC +- A **proposer** that listens for pending tx’s, creates collations, and submits them to the SMC - Ability to inspect the shard states and visualize the working system locally @@ -158,8 +158,8 @@ Our implementation revolves around 5 core components: - A **locally-running geth node** that spins up an instance of the Ethereum blockchain and mines on the Proof of Work chain - A **Sharding Manager Contract (SMC)** that is deployed onto this blockchain instance - A **sharding node** that connects to the running geth node through JSON-RPC, provides bindings to the SMC -- A **notary protocol** that allows users to stake ETH into the SMC and be selected as a notary in a certain period on a shard -- A **proposer protocol** that is tasked with processing pending tx's into collations that are then submitted to the SMC. In phase 1, proposers _do not_ execute state, but rather just serialize pending tx data into possibly valid/invalid data blobs. +- A **notary service** that allows users to stake ETH into the SMC and be selected as a notary in a certain period on a shard +- A **proposer service** that is tasked with processing pending tx's into collations that are then submitted to the SMC. In phase 1, proposers _do not_ execute state, but rather just serialize pending tx data into possibly valid/invalid data blobs. Our initial implementation will function through simple command line arguments that will allow a user running the local geth node to deposit ETH into the SMC and join as a notary that is randomly assigned to a shard in a certain period. @@ -181,13 +181,13 @@ Now, we’ll explore our architecture and implementation in detail as part of th Our Ruby Release requires users to start a local geth node running a localized, private blockchain to deploy the **SMC** into. Users can spin up a notary client as a command line entrypoint into geth while the node is running as follows: - geth sharding-notary --deposit --datadir /path/to/your/datadir --password /path/to/your/password.txt --networkid 12345 + geth sharding --actor "notary" --datadir /path/to/your/datadir --password /path/to/your/password.txt --networkid 12345 --deposit This will extract 1000ETH from the user's account balance and insert him/her into the SMC's notaries. Then, the program will listen for incoming block headers and notify the user when he/she has been selected as to vote on collations for a certain shard in a given period. Once you are selected, the sharding node will download collation information to check for data availability on vote on proposals that have been submitted via the `addHeader` function on the SMC. Users can also run a proposer client that is tasked with processing transactions into collations and submitting them to the SMC via the `addHeader` function. - geth sharding-proposer --datadir /path/to/your/datadir --password /path/to/your/password.txt --networkid 12345 + geth sharding --actor "proposer" --datadir /path/to/your/datadir --password /path/to/your/password.txt --networkid 12345 This client is tasked with processing pending transactions into blobs within collations by serializing data into collation bodies. It is responsible for submitting proposals (collation headers) to the SMC via the `addHeader` function. diff --git a/sharding/notary/service.go b/sharding/notary/service.go index 8aff199346c..f1123f2985d 100644 --- a/sharding/notary/service.go +++ b/sharding/notary/service.go @@ -37,6 +37,5 @@ func (n *Notary) Start() error { // Stop the main loop for notarizing collations. func (n *Notary) Stop() error { log.Info("Stopping notary service") - // TODO: Propose collations. return nil } diff --git a/sharding/observer/service.go b/sharding/observer/service.go new file mode 100644 index 00000000000..21da01fd5e6 --- /dev/null +++ b/sharding/observer/service.go @@ -0,0 +1,33 @@ +// Package observer launches a service attached to the sharding node +// that simply observes activity across the sharded Ethereum network. +package observer + +import ( + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/sharding/node" + cli "gopkg.in/urfave/cli.v1" +) + +// Observer holds functionality required to run an observer service +// in a sharded system. Must satisfy the Service interface defined in +// sharding/service.go. +type Observer struct { + node node.Node +} + +// NewObserver creates a new observer instance. +func NewObserver(ctx *cli.Context, node node.Node) (*Observer, error) { + return &Observer{node}, nil +} + +// Start the main routine for an observer. +func (o *Observer) Start() error { + log.Info("Starting shard observer service") + return nil +} + +// Stop the main loop for observing the shard network. +func (o *Observer) Stop() error { + log.Info("Stopping shard observer service") + return nil +} From 6093a615e5cd2aa4ae58b39ef222b4c85a2c4ba7 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 22 May 2018 12:45:27 -0400 Subject: [PATCH 12/16] sharding: edit documentation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1ccb22e3b9e..fcf23ea67ed 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ geth sharding --actor "notary" --deposit --datadir /path/to/your/datadir --passw This will extract 1000ETH from your account balance and insert you into the SMC's notaries. Then, the program will listen for incoming block headers and notify you when you have been selected as to vote on proposals for a certain shard in a given period. Once you are selected, your sharding node will download collation information to check for data availability on vote on proposals that have been submitted via the `addHeader` function on the SMC. -Concurrently, you will need to run another node that is tasked with processing transactions into collations and submitting them to the SMC via the `addHeader` function. +Concurrently, you will need to run another service that is tasked with processing transactions into collations and submitting them to the SMC via the `addHeader` function. ## Running a Collation Proposal Node From af1554370e6c64fdfe1797f73a614b777ca10884 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 22 May 2018 16:12:02 -0600 Subject: [PATCH 13/16] sharding: addressed comments, linters pass --- cmd/geth/shardingcmd.go | 19 +++++++++++-------- sharding/interfaces.go | 6 +----- sharding/node/node.go | 6 +++--- sharding/notary/service.go | 3 +-- sharding/observer/service.go | 3 +-- sharding/proposer/service.go | 3 +-- 6 files changed, 18 insertions(+), 22 deletions(-) diff --git a/cmd/geth/shardingcmd.go b/cmd/geth/shardingcmd.go index 4e8cff57b10..ce92d0976ab 100644 --- a/cmd/geth/shardingcmd.go +++ b/cmd/geth/shardingcmd.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/sharding" "github.com/ethereum/go-ethereum/sharding/node" "github.com/ethereum/go-ethereum/sharding/notary" "github.com/ethereum/go-ethereum/sharding/observer" @@ -13,7 +14,7 @@ import ( var ( shardingCommand = cli.Command{ - Action: utils.MigrateFlags(sharding), + Action: utils.MigrateFlags(shardingCmd), Name: "sharding", Usage: "Start a sharding-enabled node", ArgsUsage: "[endpoint]", @@ -25,13 +26,15 @@ Launches a sharding node that manages services related to submitting collations } ) -// sharding is the main cmd line entry point for starting a sharding-enabled node. +// shardingCmd is the main cmd line entry point for starting a sharding-enabled node. // A sharding node launches a suite of services including notary services, // proposer services, and a shardp2p protocol. -func sharding(ctx *cli.Context) error { +func shardingCmd(ctx *cli.Context) error { // configures a sharding-enabled node using the cli's context. shardingNode := node.NewNode(ctx) - registerShardingServices(shardingNode) + if err := registerShardingServices(shardingNode); err != nil { + return fmt.Errorf("could not start sharding node: %v", err) + } defer shardingNode.Close() // starts a connection to a geth node and kicks off every registered service. return shardingNode.Start() @@ -44,13 +47,13 @@ func sharding(ctx *cli.Context) error { func registerShardingServices(n node.Node) error { actorFlag := n.Context().GlobalString(utils.ActorFlag.Name) - err := n.Register(func(ctx *cli.Context) (sharding.Service, error) { + err := n.Register(func() (sharding.Service, error) { if actorFlag == "notary" { - return notary.NewNotary(n.Context(), n) + return notary.NewNotary(n) } else if actorFlag == "proposer" { - return proposer.NewProposer(n.Context(), n) + return proposer.NewProposer(n) } - return observer.NewObserver(n.Context(), n) + return observer.NewObserver(n) }) if err != nil { diff --git a/sharding/interfaces.go b/sharding/interfaces.go index a75a04ca625..1532bf789ac 100644 --- a/sharding/interfaces.go +++ b/sharding/interfaces.go @@ -1,9 +1,5 @@ package sharding -import ( - cli "gopkg.in/urfave/cli.v1" -) - // Service defines items that can be registered into a sharding node's serviceFuncs. // // life-cycle management is delegated to the sharding node. The service is allowed to @@ -21,4 +17,4 @@ type Service interface { // ServiceConstructor defines the callback passed in when registering a service // to a sharding node. -type ServiceConstructor func(ctx *cli.Context) (Service, error) +type ServiceConstructor func() (Service, error) diff --git a/sharding/node/node.go b/sharding/node/node.go index d9e2188644a..d539f68f361 100644 --- a/sharding/node/node.go +++ b/sharding/node/node.go @@ -103,7 +103,7 @@ func (n *shardingNode) Start() error { // Starts every service attached to the sharding node. for _, serviceFunc := range n.serviceFuncs { // Initializes each service by passing in the node's cli context. - service, err := serviceFunc(n.ctx) + service, err := serviceFunc() if err != nil { return err } @@ -154,10 +154,10 @@ func (n *shardingNode) configShardingNode() error { // and ShardP2P are examples of services. The rationale behind this is that the // sharding node should know very little about the functioning of its underlying // services as they should be extensible. -func (n *shardingNode) Register(service sharding.ServiceConstructor) error { +func (n *shardingNode) Register(constructor sharding.ServiceConstructor) error { n.lock.Lock() defer n.lock.Unlock() - n.serviceFuncs = append(n.serviceFuncs, service) + n.serviceFuncs = append(n.serviceFuncs, constructor) return nil } diff --git a/sharding/notary/service.go b/sharding/notary/service.go index f1123f2985d..749cd24e271 100644 --- a/sharding/notary/service.go +++ b/sharding/notary/service.go @@ -3,7 +3,6 @@ package notary import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/sharding/node" - cli "gopkg.in/urfave/cli.v1" ) // Notary holds functionality required to run a collation notary @@ -14,7 +13,7 @@ type Notary struct { } // NewNotary creates a new notary instance. -func NewNotary(ctx *cli.Context, node node.Node) (*Notary, error) { +func NewNotary(node node.Node) (*Notary, error) { return &Notary{node}, nil } diff --git a/sharding/observer/service.go b/sharding/observer/service.go index 21da01fd5e6..c5bed36e917 100644 --- a/sharding/observer/service.go +++ b/sharding/observer/service.go @@ -5,7 +5,6 @@ package observer import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/sharding/node" - cli "gopkg.in/urfave/cli.v1" ) // Observer holds functionality required to run an observer service @@ -16,7 +15,7 @@ type Observer struct { } // NewObserver creates a new observer instance. -func NewObserver(ctx *cli.Context, node node.Node) (*Observer, error) { +func NewObserver(node node.Node) (*Observer, error) { return &Observer{node}, nil } diff --git a/sharding/proposer/service.go b/sharding/proposer/service.go index a1dccd2eda0..afa490bbaee 100644 --- a/sharding/proposer/service.go +++ b/sharding/proposer/service.go @@ -3,7 +3,6 @@ package proposer import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/sharding/node" - cli "gopkg.in/urfave/cli.v1" ) // Proposer holds functionality required to run a collation proposer @@ -16,7 +15,7 @@ type Proposer struct { // NewProposer creates a struct instance. It is initialized and // registered as a service upon start of a sharding node. // Has access to the public methods of this node. -func NewProposer(ctx *cli.Context, node node.Node) (*Proposer, error) { +func NewProposer(node node.Node) (*Proposer, error) { return &Proposer{node}, nil } From 47635258d1395e5fb11f46f157433ba27fd1d15d Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 22 May 2018 16:20:37 -0600 Subject: [PATCH 14/16] sharding: address terence comment --- sharding/node/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sharding/node/node.go b/sharding/node/node.go index d539f68f361..7b40c10e5b3 100644 --- a/sharding/node/node.go +++ b/sharding/node/node.go @@ -102,7 +102,7 @@ func (n *shardingNode) Start() error { // Starts every service attached to the sharding node. for _, serviceFunc := range n.serviceFuncs { - // Initializes each service by passing in the node's cli context. + // Initializes each service. service, err := serviceFunc() if err != nil { return err From 98606d56bad5ef8e85c9848ba0b734047336962b Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 22 May 2018 18:35:03 -0600 Subject: [PATCH 15/16] sharding: address comments, elim string references --- cmd/geth/shardingcmd.go | 5 ++++- cmd/utils/flags.go | 2 +- sharding/node/node.go | 10 ++++------ sharding/node/utils.go | 12 ++++-------- 4 files changed, 13 insertions(+), 16 deletions(-) diff --git a/cmd/geth/shardingcmd.go b/cmd/geth/shardingcmd.go index ce92d0976ab..f1edc0ff33d 100644 --- a/cmd/geth/shardingcmd.go +++ b/cmd/geth/shardingcmd.go @@ -31,7 +31,10 @@ Launches a sharding node that manages services related to submitting collations // proposer services, and a shardp2p protocol. func shardingCmd(ctx *cli.Context) error { // configures a sharding-enabled node using the cli's context. - shardingNode := node.NewNode(ctx) + shardingNode, err := node.NewNode(ctx) + if err != nil { + return fmt.Errorf("could not initialize node instance: %v", err) + } if err := registerShardingServices(shardingNode); err != nil { return fmt.Errorf("could not start sharding node: %v", err) } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 087ee74ba98..47d31729e81 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -540,7 +540,7 @@ var ( } ActorFlag = cli.StringFlag{ Name: "actor", - Usage: `use the --actor "notary" or --actor "proposer" to start a notary or proposer service in the sharding node. If omitted, the sharding node registers an Observer service that simply observes the activity in the sharded network`, + Usage: `use the --actor notary or --actor proposer to start a notary or proposer service in the sharding node. If omitted, the sharding node registers an Observer service that simply observes the activity in the sharded network`, } ) diff --git a/sharding/node/node.go b/sharding/node/node.go index 7b40c10e5b3..ca3407ed609 100644 --- a/sharding/node/node.go +++ b/sharding/node/node.go @@ -55,21 +55,19 @@ type shardingNode struct { ctx *cli.Context // Command line context. smc *contracts.SMC // The deployed sharding management contract. rpcClient *rpc.Client // The RPC client connection to the main geth node. - lock sync.RWMutex // Mutex lock for concurrency management. + lock sync.Mutex // Mutex lock for concurrency management. serviceFuncs []sharding.ServiceConstructor // Stores an array of service callbacks to start upon running. } // NewNode setups the sharding config, registers the services required // by the sharded system. -func NewNode(ctx *cli.Context) Node { +func NewNode(ctx *cli.Context) (Node, error) { c := &shardingNode{ctx: ctx} - // Sets up all configuration options based on cli flags. if err := c.configShardingNode(); err != nil { - panic(err) // TODO(rauljordan): handle this + return nil, err } - - return c + return c, nil } // Start is the main entrypoint of a sharding node. It starts off every service diff --git a/sharding/node/utils.go b/sharding/node/utils.go index 3f6b7cdae9f..1838d99739e 100644 --- a/sharding/node/utils.go +++ b/sharding/node/utils.go @@ -33,11 +33,11 @@ func initSMC(n *shardingNode) (*contracts.SMC, error) { // TODO: Separate contract deployment from the sharding node. It would only need to be deployed // once on the mainnet, so this code would not need to ship with the node. if len(b) == 0 { - log.Info(fmt.Sprintf("No sharding manager contract found at %s. Deploying new contract.", sharding.ShardingManagerAddress.String())) + log.Info(fmt.Sprintf("No sharding manager contract found at %s. Deploying new contract.", sharding.ShardingManagerAddress)) txOps, err := n.CreateTXOpts(big.NewInt(0)) if err != nil { - return nil, fmt.Errorf("unable to intiate the transaction: %v", err) + return nil, fmt.Errorf("unable to initiate the transaction: %v", err) } addr, tx, contract, err := contracts.DeploySMC(txOps, n.client) @@ -52,13 +52,9 @@ func initSMC(n *shardingNode) (*contracts.SMC, error) { time.Sleep(1 * time.Second) } - log.Info(fmt.Sprintf("New contract deployed at %s", addr.String())) + log.Info(fmt.Sprintf("New contract deployed at %s", addr)) return contract, nil } - contract, err := contracts.NewSMC(sharding.ShardingManagerAddress, n.client) - if err != nil { - // TODO(terenc3t): handle this - } - return contract, nil + return contracts.NewSMC(sharding.ShardingManagerAddress, n.client) } From 90fa357602f36634df62d4ade4448aa08d46175e Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 22 May 2018 18:36:35 -0600 Subject: [PATCH 16/16] sharding: return err --- sharding/node/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sharding/node/node.go b/sharding/node/node.go index ca3407ed609..a5eec20fe45 100644 --- a/sharding/node/node.go +++ b/sharding/node/node.go @@ -136,7 +136,7 @@ func (n *shardingNode) configShardingNode() error { scryptN, scryptP, keydir, err := config.AccountConfig() if err != nil { - panic(err) // TODO(prestonvanloon): handle this. + return err } ks := keystore.NewKeyStore(keydir, scryptN, scryptP)