Skip to content

Commit

Permalink
Improve E2E to be more consistent with timing, and allow for custom f…
Browse files Browse the repository at this point in the history 
…lags (#4620)

* Add committees helper, benchmark, results show 62ms for 8k validators which was previously 4 minutes

* Add regression test with same data

* fix epoch conversion

* lint

* undo and lint

* Begin work on adding mainnet config benchmark

* Try more to get mainnet e2e

* Try to fix delay

* Get past chainstart on e2e

* Try to fix flaky

* Get demo config working

* Remove unneeded changes

* Change how flags are enabled

* Lower shard count

* Temp skip

* Fix e2e

* Fix testing to run until last epoch

* Fix

* Add ending time log and remove att cache flag

* Fix ordering

* Reenable flag

* Change ports from default

* Add no log for if there are no err logs

* Add block evaluator

* Try to improve evaluators

* Progress on attestation evaluator

* Remove attestation evaluator

* Fix e2e

Co-authored-by: Preston Van Loon <preston@prysmaticlabs.com>
  • Loading branch information
@0xKiwi @prestonvanloon @nisdas
2 people authored and nisdas committed Jan 25, 2020
1 parent 1562d32 commit fa2acb3
Show file tree
Hide file tree
Showing 7 changed files with 96 additions and 79 deletions.
41 changes: 16 additions & 25 deletions endtoend/beacon_node.go
View file
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/pkg/errors"
ev "github.com/prysmaticlabs/prysm/endtoend/evaluators"
"github.com/prysmaticlabs/prysm/shared/params"
)

type beaconNodeInfo struct {
Expand All @@ -28,12 +29,12 @@ type beaconNodeInfo struct {
}

type end2EndConfig struct {
minimalConfig bool
beaconFlags []string
validatorFlags []string
tmpPath string
epochsToRun uint64
numValidators uint64
numBeaconNodes uint64
enableSSZCache bool
contractAddr common.Address
evaluators []ev.Evaluator
}
Expand Down Expand Up @@ -69,31 +70,21 @@ func startNewBeaconNode(t *testing.T, config *end2EndConfig, beaconNodes []*beac

args := []string{
"--no-genesis-delay",
"--verbosity=debug",
"--force-clear-db",
"--no-discovery",
"--new-cache",
"--enable-shuffled-index-cache",
"--enable-skip-slots-cache",
"--enable-attestation-cache",
"--http-web3provider=http://127.0.0.1:8545",
"--web3provider=ws://127.0.0.1:8546",
"--http-web3provider=http://127.0.0.1:8745",
"--web3provider=ws://127.0.0.1:8746",
fmt.Sprintf("--datadir=%s/eth2-beacon-node-%d", tmpPath, index),
fmt.Sprintf("--deposit-contract=%s", config.contractAddr.Hex()),
fmt.Sprintf("--rpc-port=%d", 4000+index),
fmt.Sprintf("--p2p-udp-port=%d", 12000+index),
fmt.Sprintf("--p2p-tcp-port=%d", 13000+index),
fmt.Sprintf("--monitoring-port=%d", 8080+index),
fmt.Sprintf("--grpc-gateway-port=%d", 3200+index),
fmt.Sprintf("--rpc-port=%d", 4200+index),
fmt.Sprintf("--p2p-udp-port=%d", 12200+index),
fmt.Sprintf("--p2p-tcp-port=%d", 13200+index),
fmt.Sprintf("--monitoring-port=%d", 8280+index),
fmt.Sprintf("--grpc-gateway-port=%d", 3400+index),
fmt.Sprintf("--contract-deployment-block=%d", 0),
fmt.Sprintf("--rpc-max-page-size=%d", params.BeaconConfig().MinGenesisActiveValidatorCount),
}

if config.minimalConfig {
args = append(args, "--minimal-config")
}
if config.enableSSZCache {
args = append(args, "--enable-ssz-cache")
}
args = append(args, config.beaconFlags...)

// After the first node is made, have all following nodes connect to all previously made nodes.
if index >= 1 {
Expand All @@ -102,7 +93,7 @@ func startNewBeaconNode(t *testing.T, config *end2EndConfig, beaconNodes []*beac
}
}

t.Logf("Starting beacon chain with flags: %s", strings.Join(args, " "))
t.Logf("Starting beacon chain %d with flags: %s", index, strings.Join(args, " "))
cmd := exec.Command(binaryPath, args...)
cmd.Stdout = stdOutFile
cmd.Stderr = stdOutFile
Expand All @@ -122,9 +113,9 @@ func startNewBeaconNode(t *testing.T, config *end2EndConfig, beaconNodes []*beac
return &beaconNodeInfo{
processID: cmd.Process.Pid,
datadir: fmt.Sprintf("%s/eth2-beacon-node-%d", tmpPath, index),
rpcPort: 4000 + uint64(index),
monitorPort: 8080 + uint64(index),
grpcPort: 3200 + uint64(index),
rpcPort: 4200 + uint64(index),
monitorPort: 8280 + uint64(index),
grpcPort: 3400 + uint64(index),
multiAddr: multiAddr,
}
}
Expand Down
10 changes: 8 additions & 2 deletions endtoend/demo_e2e_test.go
View file
Expand Up @@ -14,9 +14,15 @@ func TestEndToEnd_DemoConfig(t *testing.T) {
params.UseDemoBeaconConfig()

demoConfig := &end2EndConfig{
minimalConfig: false,
beaconFlags: []string{
"--enable-ssz-cache",
"--cache-proposer-indices",
"--cache-filtered-block-tree",
"--enable-skip-slots-cache",
"--enable-attestation-cache",
},
epochsToRun: 5,
numBeaconNodes: 4,
numBeaconNodes: 2,
numValidators: params.BeaconConfig().MinGenesisActiveValidatorCount,
evaluators: []ev.Evaluator{
ev.ValidatorsAreActive,
Expand Down
50 changes: 22 additions & 28 deletions endtoend/endtoend_test.go
View file
Expand Up @@ -55,16 +55,16 @@ func runEndToEndTest(t *testing.T, config *end2EndConfig) {
if err != nil {
t.Fatal(err)
}
if err := waitForTextInFile(beaconLogFile, "Sending genesis time notification"); err != nil {
t.Fatalf("failed to find genesis in logs, this means the chain did not start: %v", err)
if err := waitForTextInFile(beaconLogFile, "Chain started within the last epoch"); err != nil {
t.Fatalf("failed to find chain start in logs, this means the chain did not start: %v", err)
}

// Failing early in case chain doesn't start.
if t.Failed() {
return
}

conn, err := grpc.Dial("127.0.0.1:4000", grpc.WithInsecure())
conn, err := grpc.Dial("127.0.0.1:4200", grpc.WithInsecure())
if err != nil {
t.Fatalf("Failed to dial: %v", err)
}
Expand All @@ -78,30 +78,24 @@ func runEndToEndTest(t *testing.T, config *end2EndConfig) {
// Small offset so evaluators perform in the middle of an epoch.
epochSeconds := params.BeaconConfig().SecondsPerSlot * params.BeaconConfig().SlotsPerEpoch
genesisTime := time.Unix(genesis.GenesisTime.Seconds+int64(epochSeconds/2), 0)
currentEpoch := uint64(0)
ticker := GetEpochTicker(genesisTime, epochSeconds)
for c := range ticker.C() {
if c >= config.epochsToRun || t.Failed() {
ticker.Done()
break
}

for currentEpoch := range ticker.C() {
for _, evaluator := range config.evaluators {
// Only run if the policy says so.
if !evaluator.Policy(currentEpoch) {
continue
}
t.Run(fmt.Sprintf(evaluator.Name, currentEpoch), func(t *testing.T) {
if err := evaluator.Evaluation(beaconClient); err != nil {
t.Fatalf("evaluation failed for epoch %d: %v", currentEpoch, err)
t.Errorf("evaluation failed for epoch %d: %v", currentEpoch, err)
}
})
}
currentEpoch++
}

if currentEpoch < config.epochsToRun {
t.Fatalf("Test ended prematurely, only reached epoch %d", currentEpoch)
if t.Failed() || currentEpoch >= config.epochsToRun {
ticker.Done()
break
}
}
}

Expand Down Expand Up @@ -151,22 +145,21 @@ func killProcesses(t *testing.T, pIDs []int) {
}

func logOutput(t *testing.T, tmpPath string, config *end2EndConfig) {
if t.Failed() {
// Log out errors from beacon chain nodes.
for i := uint64(0); i < config.numBeaconNodes; i++ {
beaconLogFile, err := os.Open(path.Join(tmpPath, fmt.Sprintf(beaconNodeLogFileName, i)))
if err != nil {
t.Fatal(err)
}
logErrorOutput(t, beaconLogFile, "beacon chain node", i)
// Log out errors from beacon chain nodes.
for i := uint64(0); i < config.numBeaconNodes; i++ {
beaconLogFile, err := os.Open(path.Join(tmpPath, fmt.Sprintf(beaconNodeLogFileName, i)))
if err != nil {
t.Fatal(err)
}
logErrorOutput(t, beaconLogFile, "beacon chain node", i)

validatorLogFile, err := os.Open(path.Join(tmpPath, fmt.Sprintf(validatorLogFileName, i)))
if err != nil {
t.Fatal(err)
}
logErrorOutput(t, validatorLogFile, "validator client", i)
validatorLogFile, err := os.Open(path.Join(tmpPath, fmt.Sprintf(validatorLogFileName, i)))
if err != nil {
t.Fatal(err)
}
logErrorOutput(t, validatorLogFile, "validator client", i)
}
t.Logf("Ending time: %s\n", time.Now().String())
}

func logErrorOutput(t *testing.T, file *os.File, title string, index uint64) {
Expand All @@ -181,6 +174,7 @@ func logErrorOutput(t *testing.T, file *os.File, title string, index uint64) {
}

if len(errorLines) < 1 {
t.Logf("No error logs detected for %s %d", title, index)
return
}

Expand Down
4 changes: 3 additions & 1 deletion endtoend/eth1.go
View file
Expand Up @@ -45,9 +45,11 @@ func startEth1(t *testing.T, tmpPath string) (common.Address, string, int) {
"--rpcaddr=0.0.0.0",
"--rpccorsdomain=\"*\"",
"--rpcvhosts=\"*\"",
"--rpcport=8745",
"--ws",
"--wsaddr=0.0.0.0",
"--wsorigins=\"*\"",
"--wsport=8746",
"--dev",
"--dev.period=0",
"--ipcdisable",
Expand All @@ -68,7 +70,7 @@ func startEth1(t *testing.T, tmpPath string) (common.Address, string, int) {
}

// Connect to the started geth dev chain.
client, err := rpc.DialHTTP("http://127.0.0.1:8545")
client, err := rpc.DialHTTP("http://127.0.0.1:8745")
if err != nil {
t.Fatalf("Failed to connect to ipc: %v", err)
}
Expand Down
39 changes: 28 additions & 11 deletions endtoend/evaluators/validator.go
View file
Expand Up @@ -44,7 +44,9 @@ func afterNthEpoch(afterEpoch uint64) func(uint64) bool {

func validatorsAreActive(client eth.BeaconChainClient) error {
// Balances actually fluctuate but we just want to check initial balance.
validatorRequest := &eth.ListValidatorsRequest{}
validatorRequest := &eth.ListValidatorsRequest{
PageSize: int32(params.BeaconConfig().MinGenesisActiveValidatorCount),
}
validators, err := client.ListValidators(context.Background(), validatorRequest)
if err != nil {
return errors.Wrap(err, "failed to get validators")
Expand All @@ -56,24 +58,39 @@ func validatorsAreActive(client eth.BeaconChainClient) error {
return fmt.Errorf("expected validator count to be %d, recevied %d", expectedCount, receivedCount)
}

effBalanceLowCount := 0
activeEpochWrongCount := 0
exitEpochWrongCount := 0
withdrawEpochWrongCount := 0
for _, item := range validators.ValidatorList {
if item.Validator.EffectiveBalance < params.BeaconConfig().MaxEffectiveBalance {
effBalanceLowCount++
}
if item.Validator.ActivationEpoch != 0 {
return fmt.Errorf("expected genesis validator epoch to be 0, received %d", item.Validator.ActivationEpoch)
activeEpochWrongCount++
}
if item.Validator.ExitEpoch != params.BeaconConfig().FarFutureEpoch {
return fmt.Errorf("expected genesis validator exit epoch to be far future, received %d", item.Validator.ExitEpoch)
exitEpochWrongCount++
}
if item.Validator.WithdrawableEpoch != params.BeaconConfig().FarFutureEpoch {
return fmt.Errorf("expected genesis validator withdrawable epoch to be far future, received %d", item.Validator.WithdrawableEpoch)
}
if item.Validator.EffectiveBalance != params.BeaconConfig().MaxEffectiveBalance {
return fmt.Errorf(
"expected genesis validator effective balance to be %d, received %d",
params.BeaconConfig().MaxEffectiveBalance,
item.Validator.EffectiveBalance,
)
withdrawEpochWrongCount++
}
}

if effBalanceLowCount > 0 {
return fmt.Errorf(
"%d validators did not have genesis validator effective balance of %d",
effBalanceLowCount,
params.BeaconConfig().MaxEffectiveBalance,
)
} else if activeEpochWrongCount > 0 {
return fmt.Errorf("%d validators did not have genesis validator epoch of 0", activeEpochWrongCount)
} else if exitEpochWrongCount > 0 {
return fmt.Errorf("%d validators did not have genesis validator exit epoch of far future epoch", exitEpochWrongCount)
} else if activeEpochWrongCount > 0 {
return fmt.Errorf("%d validators did not have genesis validator withdrawable epoch of far future epoch", activeEpochWrongCount)
}

return nil
}

Expand Down
13 changes: 11 additions & 2 deletions endtoend/minimal_e2e_test.go
View file
Expand Up @@ -13,10 +13,19 @@ func TestEndToEnd_MinimalConfig(t *testing.T) {
params.UseMinimalConfig()

minimalConfig := &end2EndConfig{
minimalConfig: true,
beaconFlags: []string{
"--minimal-config",
"--enable-ssz-cache",
"--cache-proposer-indices",
"--cache-filtered-block-tree",
"--enable-skip-slots-cache",
"--enable-attestation-cache",
},
validatorFlags: []string{
"--minimal-config",
},
epochsToRun: 5,
numBeaconNodes: 4,
enableSSZCache: true,
numValidators: params.BeaconConfig().MinGenesisActiveValidatorCount,
evaluators: []ev.Evaluator{
ev.ValidatorsAreActive,
Expand Down
18 changes: 8 additions & 10 deletions endtoend/validator.go
View file
Expand Up @@ -58,27 +58,26 @@ func initializeValidators(
"--force-clear-db",
fmt.Sprintf("--interop-num-validators=%d", validatorsPerNode),
fmt.Sprintf("--interop-start-index=%d", validatorsPerNode*n),
fmt.Sprintf("--monitoring-port=%d", 9080+n),
fmt.Sprintf("--monitoring-port=%d", 9280+n),
fmt.Sprintf("--datadir=%s/eth2-val-%d", tmpPath, n),
fmt.Sprintf("--beacon-rpc-provider=localhost:%d", 4000+n),
}
if config.minimalConfig {
args = append(args, "--minimal-config")
fmt.Sprintf("--beacon-rpc-provider=localhost:%d", 4200+n),
}
args = append(args, config.validatorFlags...)

cmd := exec.Command(binaryPath, args...)
cmd.Stdout = file
cmd.Stderr = file
t.Logf("Starting validator client with flags: %s", strings.Join(args, " "))
t.Logf("Starting validator client %d with flags: %s", n, strings.Join(args, " "))
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
valClients[n] = &validatorClientInfo{
processID: cmd.Process.Pid,
monitorPort: 9080 + n,
monitorPort: 9280 + n,
}
}

client, err := rpc.DialHTTP("http://127.0.0.1:8545")
client, err := rpc.DialHTTP("http://127.0.0.1:8745")
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -124,8 +123,7 @@ func initializeValidators(
t.Fatal(err)
}

// "Safe" amount of blocks to mine to make sure the deposits are seen.
if err := mineBlocks(web3, keystore, 20); err != nil {
if err := mineBlocks(web3, keystore, params.BeaconConfig().Eth1FollowDistance); err != nil {
t.Fatalf("failed to mine blocks %v", err)
}

Expand Down

0 comments on commit fa2acb3

Please sign in to comment.