diff --git a/beacon-chain/core/helpers/BUILD.bazel b/beacon-chain/core/helpers/BUILD.bazel index c7deb40878c..1dc8ea5718e 100644 --- a/beacon-chain/core/helpers/BUILD.bazel +++ b/beacon-chain/core/helpers/BUILD.bazel @@ -16,6 +16,7 @@ go_library( visibility = [ "//beacon-chain:__subpackages__", "//shared/testutil:__pkg__", + "//shared/benchutil/benchmark_files:__subpackages__", "//slasher:__subpackages__", "//tools:__subpackages__", "//validator:__subpackages__", diff --git a/beacon-chain/core/state/BUILD.bazel b/beacon-chain/core/state/BUILD.bazel index 96d4bbbae41..7ca5466256c 100644 --- a/beacon-chain/core/state/BUILD.bazel +++ b/beacon-chain/core/state/BUILD.bazel @@ -14,6 +14,7 @@ go_library( "//beacon-chain:__subpackages__", "//shared/interop:__pkg__", "//shared/testutil:__pkg__", + "//tools/benchmark-files-gen:__pkg__", "//tools/genesis-state-gen:__pkg__", ], deps = [ @@ -46,19 +47,23 @@ go_test( name = "go_default_test", size = "small", srcs = [ + "benchmarks_test.go", "skip_slot_cache_test.go", "state_test.go", "transition_test.go", ], + data = ["//shared/benchutil/benchmark_files:benchmark_data"], embed = [":go_default_library"], deps = [ "//beacon-chain/core/blocks:go_default_library", "//beacon-chain/core/helpers:go_default_library", "//proto/beacon/p2p/v1:go_default_library", + "//shared/benchutil:go_default_library", "//shared/bls:go_default_library", "//shared/featureconfig:go_default_library", "//shared/hashutil:go_default_library", "//shared/params:go_default_library", + "//shared/stateutil:go_default_library", "//shared/testutil:go_default_library", "//shared/trieutil:go_default_library", "@com_github_gogo_protobuf//proto:go_default_library", diff --git a/beacon-chain/core/state/benchmarks/BUILD.bazel b/beacon-chain/core/state/benchmarks/BUILD.bazel deleted file mode 100644 index c4922c2e5a1..00000000000 --- a/beacon-chain/core/state/benchmarks/BUILD.bazel +++ /dev/null @@ -1,28 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") - -go_test( - name = "go_default_test", - srcs = ["benchmarks_test.go"], - data = ["//beacon-chain/core/state/benchmarks/benchmark_files:benchmark_data"], - embed = [":go_default_library"], - deps = [ - "//beacon-chain/core/helpers:go_default_library", - "//beacon-chain/core/state:go_default_library", - "//proto/beacon/p2p/v1:go_default_library", - "//shared/params:go_default_library", - "//shared/stateutil:go_default_library", - "@com_github_gogo_protobuf//proto:go_default_library", - "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library", - "@com_github_prysmaticlabs_go_ssz//:go_default_library", - "@io_bazel_rules_go//go/tools/bazel:go_default_library", - ], -) - -go_library( - name = "go_default_library", - testonly = True, - srcs = ["config.go"], - importpath = "github.com/prysmaticlabs/prysm/beacon-chain/core/state/benchmarks", - visibility = ["//beacon-chain/core/state/benchmarks:__subpackages__"], - deps = ["//shared/params:go_default_library"], -) diff --git a/beacon-chain/core/state/benchmarks/README.md b/beacon-chain/core/state/benchmarks/README.md deleted file mode 100644 index 96043dbaf60..00000000000 --- a/beacon-chain/core/state/benchmarks/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# Benchmarks for Prysm State Transition -This package contains the functionality needed for benchmarking Prysms state transitions, this includes its block processing (with and without caching) and epoch processing functions. There is also a benchmark for HashTreeRoot on a large beacon state. - -## Benchmark Configuration -The following configs are in `config.go`: -* `ValidatorCount`: Sets the amount of active validators to perform the benchmarks with. Default is 65536. -* `AttestationsPerEpoch`: Sets the amount of attestations per epoch for the benchmarks to perform with, this affects the amount of attestations in a full block and the amount of attestations per epoch in the state for the `ProcessEpoch` and `HashTreeRoot` benchmark. Default is 128. - -## Generating new SSZ files -Due to the sheer size of the benchmarking configurations (65536 validators), the files used for benchmarking are pregenerated so there's no wasted computations on generating a genesis state with 65536 validators. This should only be needed if there is a breaking spec change and the tests fail from SSZ issues. - -To generate new files to use for benchmarking, run the below command in the root of Prysm. -``` -go run beacon-chain/core/state/benchmarks/benchmark_files/generate_bench_files.go -``` - -Bazel does not allow writing to the project directory, so running with `go run` is needed. - -## Running the benchmarks -To run the ExecuteStateTransition benchmark: - -```bazel test //beacon-chain/core/state/benchmarks:go_default_test --test_filter=BenchmarkExecuteStateTransition_FullBlock --test_arg=-test.bench=BenchmarkExecuteStateTransition_FullBlock``` - -To run the ExecuteStateTransition (with cache) benchmark: - -```bazel test //beacon-chain/core/state/benchmarks:go_default_test --test_filter=BenchmarkExecuteStateTransition_WithCache --test_arg=-test.bench=BenchmarkExecuteStateTransition_WithCache``` - -To run the ProcessEpoch benchmark: - -```bazel test //beacon-chain/core/state/benchmarks:go_default_test --test_filter=BenchmarkProcessEpoch_2FullEpochs --test_arg=-test.bench=BenchmarkProcessEpoch_2FullEpochs``` - -To run the HashTreeRoot benchmark: - -```bazel test //beacon-chain/core/state/benchmarks:go_default_test --test_filter=BenchmarkHashTreeRoot_FullState --test_arg=-test.bench=BenchmarkHashTreeRoot_FullState``` - -To run the HashTreeRootState benchmark: - -```bazel test //beacon-chain/core/state/benchmarks:go_default_test --test_filter=BenchmarkHashTreeRootState_FullState --test_arg=-test.bench=BenchmarkHashTreeRootState_FullState``` - -Extra flags needed to benchmark properly: - -```--nocache_test_results --test_arg=-test.v --test_timeout=2000 --test_arg=-test.cpuprofile=/tmp/cpu.profile --test_arg=-test.memprofile=/tmp/mem.profile --test_output=streamed``` - -## Current Results as of November 2019 -``` -BenchmarkExecuteStateTransition-4 20 33020593584 ns/op -BenchmarkExecuteStateTransition_WithCache-4 20 21272276477 ns/op -BenchmarkProcessEpoch_2FullEpochs-4 5 158161708836 ns/op -BenchmarkHashTreeRoot_FullState-4 50 1509721280 ns/op -BenchmarkHashTreeRootState_FullState-4 50 67622586 ns/op -``` \ No newline at end of file diff --git a/beacon-chain/core/state/benchmarks/benchmark_files/bState1Epoch-128Atts-65536Vals.ssz b/beacon-chain/core/state/benchmarks/benchmark_files/bState1Epoch-128Atts-65536Vals.ssz deleted file mode 100644 index 400afe51d5f..00000000000 Binary files a/beacon-chain/core/state/benchmarks/benchmark_files/bState1Epoch-128Atts-65536Vals.ssz and /dev/null differ diff --git a/beacon-chain/core/state/benchmarks/benchmark_files/bState2Epochs-128Atts-65536Vals.ssz b/beacon-chain/core/state/benchmarks/benchmark_files/bState2Epochs-128Atts-65536Vals.ssz deleted file mode 100644 index 1e7ca818d93..00000000000 Binary files a/beacon-chain/core/state/benchmarks/benchmark_files/bState2Epochs-128Atts-65536Vals.ssz and /dev/null differ diff --git a/beacon-chain/core/state/benchmarks/benchmark_files/fullBlock-128Atts-65536Vals.ssz b/beacon-chain/core/state/benchmarks/benchmark_files/fullBlock-128Atts-65536Vals.ssz deleted file mode 100644 index 1288778b55b..00000000000 Binary files a/beacon-chain/core/state/benchmarks/benchmark_files/fullBlock-128Atts-65536Vals.ssz and /dev/null differ diff --git a/beacon-chain/core/state/benchmarks/config.go b/beacon-chain/core/state/benchmarks/config.go deleted file mode 100644 index cb7f006a25b..00000000000 --- a/beacon-chain/core/state/benchmarks/config.go +++ /dev/null @@ -1,48 +0,0 @@ -package benchmarks - -import ( - "fmt" - - "github.com/prysmaticlabs/prysm/shared/params" -) - -// ValidatorCount is for declaring how many validators the benchmarks will be -// performed with. Default is 65536 or 2M ETH staked. -var ValidatorCount = uint64(65536) - -// AttestationsPerEpoch represents the requested amount attestations in an epoch. -// This affects the amount of attestations in a fully attested for block and the amount -// of attestations in the state per epoch, so a full 2 epochs should result in twice -// this amount of attestations in the state. Default is 128. -var AttestationsPerEpoch = uint64(128) - -// GenesisFileName is the generated genesis beacon state file name. -var GenesisFileName = fmt.Sprintf("benchmark_files/bStateGenesis-%dAtts-%dVals.ssz", AttestationsPerEpoch, ValidatorCount) - -// BState1EpochFileName is the generated beacon state after 1 skipped epoch file name. -var BState1EpochFileName = fmt.Sprintf("benchmark_files/bState1Epoch-%dAtts-%dVals.ssz", AttestationsPerEpoch, ValidatorCount) - -// BState2EpochFileName is the generated beacon state after 2 full epochs file name. -var BState2EpochFileName = fmt.Sprintf("benchmark_files/bState2Epochs-%dAtts-%dVals.ssz", AttestationsPerEpoch, ValidatorCount) - -// FullBlockFileName is the generated full block file name. -var FullBlockFileName = fmt.Sprintf("benchmark_files/fullBlock-%dAtts-%dVals.ssz", AttestationsPerEpoch, ValidatorCount) - -// FilePath prefixes the file path to the file names. -func FilePath(fileName string) string { - return fmt.Sprintf("beacon-chain/core/state/benchmarks/%s", fileName) -} - -// SetConfig changes the beacon config to match the requested amount of -// attestations set to AttestationsPerEpoch. -func SetConfig() { - maxAtts := AttestationsPerEpoch - slotsPerEpoch := params.BeaconConfig().SlotsPerEpoch - committeeSize := (ValidatorCount / slotsPerEpoch) / (maxAtts / slotsPerEpoch) - c := params.BeaconConfig() - c.PersistentCommitteePeriod = 0 - c.MinValidatorWithdrawabilityDelay = 0 - c.TargetCommitteeSize = committeeSize - c.MaxAttestations = maxAtts - params.OverrideBeaconConfig(c) -} diff --git a/beacon-chain/core/state/benchmarks/benchmarks_test.go b/beacon-chain/core/state/benchmarks_test.go similarity index 58% rename from beacon-chain/core/state/benchmarks/benchmarks_test.go rename to beacon-chain/core/state/benchmarks_test.go index 19185d4f9d8..6dce186f2a8 100644 --- a/beacon-chain/core/state/benchmarks/benchmarks_test.go +++ b/beacon-chain/core/state/benchmarks_test.go @@ -1,17 +1,15 @@ -package benchmarks +package state import ( "context" - "io/ioutil" "testing" - "github.com/bazelbuild/rules_go/go/tools/bazel" "github.com/gogo/protobuf/proto" - ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1" "github.com/prysmaticlabs/go-ssz" "github.com/prysmaticlabs/prysm/beacon-chain/core/helpers" - "github.com/prysmaticlabs/prysm/beacon-chain/core/state" pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" + "github.com/prysmaticlabs/prysm/shared/benchutil" + "github.com/prysmaticlabs/prysm/shared/featureconfig" "github.com/prysmaticlabs/prysm/shared/params" "github.com/prysmaticlabs/prysm/shared/stateutil" ) @@ -19,30 +17,29 @@ import ( var runAmount = 25 func TestBenchmarkExecuteStateTransition(t *testing.T) { - t.Skip("TODO(4098): Regenerate test data with v0.9.2 spec") - SetConfig() - beaconState, err := beaconState1Epoch() + benchutil.SetBenchmarkConfig() + beaconState, err := benchutil.PreGenState1Epoch() if err != nil { t.Fatal(err) } - block, err := fullBlock() + block, err := benchutil.PreGenFullBlock() if err != nil { t.Fatal(err) } - if _, err := state.ExecuteStateTransition(context.Background(), beaconState, block); err != nil { + if _, err := ExecuteStateTransition(context.Background(), beaconState, block); err != nil { t.Fatalf("failed to process block, benchmarks will fail: %v", err) } } func BenchmarkExecuteStateTransition_FullBlock(b *testing.B) { - SetConfig() - beaconState, err := beaconState1Epoch() + benchutil.SetBenchmarkConfig() + beaconState, err := benchutil.PreGenState1Epoch() if err != nil { b.Fatal(err) } cleanStates := clonedStates(beaconState) - block, err := fullBlock() + block, err := benchutil.PreGenFullBlock() if err != nil { b.Fatal(err) } @@ -50,21 +47,26 @@ func BenchmarkExecuteStateTransition_FullBlock(b *testing.B) { b.N = runAmount b.ResetTimer() for i := 0; i < b.N; i++ { - if _, err := state.ExecuteStateTransition(context.Background(), cleanStates[i], block); err != nil { + if _, err := ExecuteStateTransition(context.Background(), cleanStates[i], block); err != nil { b.Fatal(err) } } } func BenchmarkExecuteStateTransition_WithCache(b *testing.B) { - SetConfig() + config := &featureconfig.Flags{ + EnableProposerIndexCache: true, + EnableAttestationCache: true, + } + featureconfig.Init(config) + benchutil.SetBenchmarkConfig() - beaconState, err := beaconState1Epoch() + beaconState, err := benchutil.PreGenState1Epoch() if err != nil { b.Fatal(err) } cleanStates := clonedStates(beaconState) - block, err := fullBlock() + block, err := benchutil.PreGenFullBlock() if err != nil { b.Fatal(err) } @@ -78,22 +80,27 @@ func BenchmarkExecuteStateTransition_WithCache(b *testing.B) { } beaconState.Slot = currentSlot // Run the state transition once to populate the cache. - if _, err := state.ExecuteStateTransition(context.Background(), beaconState, block); err != nil { + if _, err := ExecuteStateTransition(context.Background(), beaconState, block); err != nil { b.Fatalf("failed to process block, benchmarks will fail: %v", err) } b.N = runAmount b.ResetTimer() for i := 0; i < b.N; i++ { - if _, err := state.ExecuteStateTransition(context.Background(), cleanStates[i], block); err != nil { + if _, err := ExecuteStateTransition(context.Background(), cleanStates[i], block); err != nil { b.Fatalf("failed to process block, benchmarks will fail: %v", err) } } } func BenchmarkProcessEpoch_2FullEpochs(b *testing.B) { - SetConfig() - beaconState, err := beaconState2FullEpochs() + config := &featureconfig.Flags{ + EnableProposerIndexCache: true, + EnableAttestationCache: true, + } + featureconfig.Init(config) + benchutil.SetBenchmarkConfig() + beaconState, err := benchutil.PreGenState2FullEpochs() if err != nil { b.Fatal(err) } @@ -113,14 +120,15 @@ func BenchmarkProcessEpoch_2FullEpochs(b *testing.B) { for i := 0; i < b.N; i++ { // ProcessEpochPrecompute is the optimized version of process epoch. It's enabled by default // at run time. - if _, err := state.ProcessEpochPrecompute(context.Background(), cleanStates[i]); err != nil { + b.Log(i) + if _, err := ProcessEpochPrecompute(context.Background(), cleanStates[i]); err != nil { b.Fatal(err) } } } func BenchmarkHashTreeRoot_FullState(b *testing.B) { - beaconState, err := beaconState2FullEpochs() + beaconState, err := benchutil.PreGenState2FullEpochs() if err != nil { b.Fatal(err) } @@ -135,7 +143,7 @@ func BenchmarkHashTreeRoot_FullState(b *testing.B) { } func BenchmarkHashTreeRootState_FullState(b *testing.B) { - beaconState, err := beaconState2FullEpochs() + beaconState, err := benchutil.PreGenState2FullEpochs() if err != nil { b.Fatal(err) } @@ -161,51 +169,3 @@ func clonedStates(beaconState *pb.BeaconState) []*pb.BeaconState { } return clonedStates } - -func beaconState1Epoch() (*pb.BeaconState, error) { - path, err := bazel.Runfile(BState1EpochFileName) - if err != nil { - return nil, err - } - beaconBytes, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - beaconState := &pb.BeaconState{} - if err := ssz.Unmarshal(beaconBytes, beaconState); err != nil { - return nil, err - } - return beaconState, nil -} - -func beaconState2FullEpochs() (*pb.BeaconState, error) { - path, err := bazel.Runfile(BState2EpochFileName) - if err != nil { - return nil, err - } - beaconBytes, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - beaconState := &pb.BeaconState{} - if err := ssz.Unmarshal(beaconBytes, beaconState); err != nil { - return nil, err - } - return beaconState, nil -} - -func fullBlock() (*ethpb.SignedBeaconBlock, error) { - path, err := bazel.Runfile(FullBlockFileName) - if err != nil { - return nil, err - } - blockBytes, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - beaconBlock := ðpb.SignedBeaconBlock{} - if err := ssz.Unmarshal(blockBytes, beaconBlock); err != nil { - return nil, err - } - return beaconBlock, nil -} diff --git a/shared/benchutil/BUILD.bazel b/shared/benchutil/BUILD.bazel new file mode 100644 index 00000000000..c5049c20af9 --- /dev/null +++ b/shared/benchutil/BUILD.bazel @@ -0,0 +1,22 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = ["pregen.go"], + data = ["//shared/benchutil/benchmark_files:benchmark_data"], + importpath = "github.com/prysmaticlabs/prysm/shared/benchutil", + visibility = ["//visibility:public"], + deps = [ + "//proto/beacon/p2p/v1:go_default_library", + "//shared/params:go_default_library", + "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library", + "@com_github_prysmaticlabs_go_ssz//:go_default_library", + "@io_bazel_rules_go//go/tools/bazel:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = ["pregen_test.go"], + embed = [":go_default_library"], +) diff --git a/shared/benchutil/README.md b/shared/benchutil/README.md new file mode 100644 index 00000000000..58880f4f561 --- /dev/null +++ b/shared/benchutil/README.md @@ -0,0 +1,48 @@ +# Benchmarks for Prysm State Transition +This package contains the functionality needed for benchmarking Prysms state transitions, this includes its block processing (with and without caching) and epoch processing functions. There is also a benchmark for HashTreeRoot on a large beacon state. + +## Benchmark Configuration +The following configs are in `config.go`: +* `ValidatorCount`: Sets the amount of active validators to perform the benchmarks with. Default is 16384. +* `AttestationsPerEpoch`: Sets the amount of attestations per epoch for the benchmarks to perform with, this affects the amount of attestations in a full block and the amount of attestations per epoch in the state for the `ProcessEpoch` and `HashTreeRoot` benchmark. Default is 128. + +## Generating new SSZ files +Due to the sheer size of the benchmarking configurations (16384 validators), the files used for benchmarking are pregenerated so there's no wasted computations on generating a genesis state with 16384 validators. This should only be needed if there is a breaking spec change and the tests fail from SSZ issues. + +To generate new files to use for benchmarking, run the below command in the root of Prysm. +``` +bazel run //tools/benchmark-files-gen -- --output-dir $PRYSMPATH/shared/benchutil/benchmark_files/ --overwrite +``` + +## Running the benchmarks +To run the ExecuteStateTransition benchmark: + +```bazel test //beacon-chain/core/state:go_default_test --test_filter=BenchmarkExecuteStateTransition_FullBlock --test_arg=-test.bench=BenchmarkExecuteStateTransition_FullBlock``` + +To run the ExecuteStateTransition (with cache) benchmark: + +```bazel test //beacon-chain/core/state:go_default_test --test_filter=BenchmarkExecuteStateTransition_WithCache --test_arg=-test.bench=BenchmarkExecuteStateTransition_WithCache``` + +To run the ProcessEpoch benchmark: + +```bazel test //beacon-chain/core/state:go_default_test --test_filter=BenchmarkProcessEpoch_2FullEpochs --test_arg=-test.bench=BenchmarkProcessEpoch_2FullEpochs``` + +To run the HashTreeRoot benchmark: + +```bazel test //beacon-chain/core/state:go_default_test --test_filter=BenchmarkHashTreeRoot_FullState --test_arg=-test.bench=BenchmarkHashTreeRoot_FullState``` + +To run the HashTreeRootState benchmark: + +```bazel test //beacon-chain/core/state:go_default_test --test_filter=BenchmarkHashTreeRootState_FullState --test_arg=-test.bench=BenchmarkHashTreeRootState_FullState``` + +Extra flags needed to benchmark properly: + +```--nocache_test_results --test_arg=-test.v --test_timeout=2000 --test_arg=-test.cpuprofile=/tmp/cpu.profile --test_arg=-test.memprofile=/tmp/mem.profile --test_output=streamed``` + +## Current Results as of January 2020 +``` +BenchmarkExecuteStateTransition_FullBlock-4 20 2031438030 ns/op +BenchmarkExecuteStateTransition_WithCache-4 20 1857290454 ns/op +BenchmarkHashTreeRoot_FullState-4 50 297655834 ns/op +BenchmarkHashTreeRootState_FullState-4 50 155535883 ns/op +``` \ No newline at end of file diff --git a/shared/benchutil/benchmark_files/BUILD.bazel b/shared/benchutil/benchmark_files/BUILD.bazel new file mode 100644 index 00000000000..9d443107053 --- /dev/null +++ b/shared/benchutil/benchmark_files/BUILD.bazel @@ -0,0 +1,10 @@ +filegroup( + name = "benchmark_data", + srcs = glob([ + "*.ssz", + ]), + visibility = [ + "//beacon-chain/core/state:__pkg__", + "//shared/benchutil:__subpackages__", + ], +) diff --git a/shared/benchutil/benchmark_files/bState1Epoch-128Atts-16384Vals.ssz b/shared/benchutil/benchmark_files/bState1Epoch-128Atts-16384Vals.ssz new file mode 100644 index 00000000000..af65a82e292 Binary files /dev/null and b/shared/benchutil/benchmark_files/bState1Epoch-128Atts-16384Vals.ssz differ diff --git a/shared/benchutil/benchmark_files/bState2Epochs-128Atts-16384Vals.ssz b/shared/benchutil/benchmark_files/bState2Epochs-128Atts-16384Vals.ssz new file mode 100644 index 00000000000..83e66a04bc8 Binary files /dev/null and b/shared/benchutil/benchmark_files/bState2Epochs-128Atts-16384Vals.ssz differ diff --git a/shared/benchutil/benchmark_files/fullBlock-128Atts-16384Vals.ssz b/shared/benchutil/benchmark_files/fullBlock-128Atts-16384Vals.ssz new file mode 100644 index 00000000000..aa0361e705e Binary files /dev/null and b/shared/benchutil/benchmark_files/fullBlock-128Atts-16384Vals.ssz differ diff --git a/shared/benchutil/pregen.go b/shared/benchutil/pregen.go new file mode 100644 index 00000000000..49751662cc1 --- /dev/null +++ b/shared/benchutil/pregen.go @@ -0,0 +1,103 @@ +package benchutil + +import ( + "fmt" + "io/ioutil" + + "github.com/bazelbuild/rules_go/go/tools/bazel" + ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1" + "github.com/prysmaticlabs/go-ssz" + pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" + "github.com/prysmaticlabs/prysm/shared/params" +) + +// ValidatorCount is for declaring how many validators the benchmarks will be +// performed with. Default is 16384 or 524K ETH staked. +var ValidatorCount = uint64(16384) + +// AttestationsPerEpoch represents the requested amount attestations in an epoch. +// This affects the amount of attestations in a fully attested for block and the amount +// of attestations in the state per epoch, so a full 2 epochs should result in twice +// this amount of attestations in the state. Default is 128. +var AttestationsPerEpoch = uint64(128) + +// GenesisFileName is the generated genesis beacon state file name. +var GenesisFileName = fmt.Sprintf("bStateGenesis-%dAtts-%dVals.ssz", AttestationsPerEpoch, ValidatorCount) + +// BState1EpochFileName is the generated beacon state after 1 skipped epoch file name. +var BState1EpochFileName = fmt.Sprintf("bState1Epoch-%dAtts-%dVals.ssz", AttestationsPerEpoch, ValidatorCount) + +// BState2EpochFileName is the generated beacon state after 2 full epochs file name. +var BState2EpochFileName = fmt.Sprintf("bState2Epochs-%dAtts-%dVals.ssz", AttestationsPerEpoch, ValidatorCount) + +// FullBlockFileName is the generated full block file name. +var FullBlockFileName = fmt.Sprintf("fullBlock-%dAtts-%dVals.ssz", AttestationsPerEpoch, ValidatorCount) + +func filePath(path string) string { + return fmt.Sprintf("shared/benchutil/benchmark_files/%s", path) +} + +// PreGenState1Epoch unmarshals the pre-generated beacon state after 1 epoch of block processing and returns it. +func PreGenState1Epoch() (*pb.BeaconState, error) { + path, err := bazel.Runfile(filePath(BState1EpochFileName)) + if err != nil { + return nil, err + } + beaconBytes, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + beaconState := &pb.BeaconState{} + if err := ssz.Unmarshal(beaconBytes, beaconState); err != nil { + return nil, err + } + return beaconState, nil +} + +// PreGenState2FullEpochs unmarshals the pre-generated beacon state after 2 epoch of full block processing and returns it. +func PreGenState2FullEpochs() (*pb.BeaconState, error) { + path, err := bazel.Runfile(filePath(BState2EpochFileName)) + if err != nil { + return nil, err + } + beaconBytes, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + beaconState := &pb.BeaconState{} + if err := ssz.Unmarshal(beaconBytes, beaconState); err != nil { + return nil, err + } + return beaconState, nil +} + +// PreGenFullBlock unmarshals the pre-generated signed beacon block containing an epochs worth of attestations and returns it. +func PreGenFullBlock() (*ethpb.SignedBeaconBlock, error) { + path, err := bazel.Runfile(filePath(FullBlockFileName)) + if err != nil { + return nil, err + } + blockBytes, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + beaconBlock := ðpb.SignedBeaconBlock{} + if err := ssz.Unmarshal(blockBytes, beaconBlock); err != nil { + return nil, err + } + return beaconBlock, nil +} + +// SetBenchmarkConfig changes the beacon config to match the requested amount of +// attestations set to AttestationsPerEpoch. +func SetBenchmarkConfig() { + maxAtts := AttestationsPerEpoch + slotsPerEpoch := params.BeaconConfig().SlotsPerEpoch + committeeSize := (ValidatorCount / slotsPerEpoch) / (maxAtts / slotsPerEpoch) + c := params.BeaconConfig() + c.PersistentCommitteePeriod = 0 + c.MinValidatorWithdrawabilityDelay = 0 + c.TargetCommitteeSize = committeeSize + c.MaxAttestations = maxAtts + params.OverrideBeaconConfig(c) +} diff --git a/shared/benchutil/pregen_test.go b/shared/benchutil/pregen_test.go new file mode 100644 index 00000000000..2d4115809c9 --- /dev/null +++ b/shared/benchutil/pregen_test.go @@ -0,0 +1,26 @@ +package benchutil + +import ( + "testing" +) + +func TestPreGenFullBlock(t *testing.T) { + _, err := PreGenFullBlock() + if err != nil { + t.Fatal(err) + } +} + +func TestPreGenState1Epoch(t *testing.T) { + _, err := PreGenFullBlock() + if err != nil { + t.Fatal(err) + } +} + +func TestPreGenState2FullEpochs(t *testing.T) { + _, err := PreGenFullBlock() + if err != nil { + t.Fatal(err) + } +} diff --git a/beacon-chain/core/state/benchmarks/benchmark_files/BUILD.bazel b/tools/benchmark-files-gen/BUILD.bazel similarity index 59% rename from beacon-chain/core/state/benchmarks/benchmark_files/BUILD.bazel rename to tools/benchmark-files-gen/BUILD.bazel index b8ad2905e0a..91703000865 100644 --- a/beacon-chain/core/state/benchmarks/benchmark_files/BUILD.bazel +++ b/tools/benchmark-files-gen/BUILD.bazel @@ -3,34 +3,26 @@ load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") go_library( name = "go_default_library", testonly = True, - srcs = ["generate_bench_files.go"], - data = glob(["*.ssz"]), - importpath = "github.com/prysmaticlabs/prysm/beacon-chain/core/state/benchmarks/benchmark_files", - visibility = ["//beacon-chain/core/state/benchmarks:__subpackages__"], + srcs = ["main.go"], + importpath = "github.com/prysmaticlabs/prysm/tools/benchmark-files-gen", + visibility = ["//visibility:private"], deps = [ "//beacon-chain/core/helpers:go_default_library", "//beacon-chain/core/state:go_default_library", - "//beacon-chain/core/state/benchmarks:go_default_library", "//proto/beacon/p2p/v1:go_default_library", + "//shared/benchutil:go_default_library", "//shared/interop:go_default_library", "//shared/params:go_default_library", "//shared/testutil:go_default_library", + "@com_github_pkg_errors//:go_default_library", "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library", "@com_github_prysmaticlabs_go_ssz//:go_default_library", ], ) -filegroup( - name = "benchmark_data", - srcs = glob([ - "*.ssz", - ]), - visibility = ["//beacon-chain/core/state/benchmarks:__subpackages__"], -) - go_binary( - name = "benchmark_files", + name = "benchmark-files-gen", testonly = True, embed = [":go_default_library"], - visibility = ["//visibility:private"], + visibility = ["//visibility:public"], ) diff --git a/beacon-chain/core/state/benchmarks/benchmark_files/generate_bench_files.go b/tools/benchmark-files-gen/main.go similarity index 61% rename from beacon-chain/core/state/benchmarks/benchmark_files/generate_bench_files.go rename to tools/benchmark-files-gen/main.go index dc2409ea27a..9b4d890ac4f 100644 --- a/beacon-chain/core/state/benchmarks/benchmark_files/generate_bench_files.go +++ b/tools/benchmark-files-gen/main.go @@ -2,43 +2,73 @@ package main import ( "context" + "flag" "io/ioutil" "log" "os" + "path" + "github.com/pkg/errors" ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1" "github.com/prysmaticlabs/go-ssz" "github.com/prysmaticlabs/prysm/beacon-chain/core/helpers" "github.com/prysmaticlabs/prysm/beacon-chain/core/state" - bench "github.com/prysmaticlabs/prysm/beacon-chain/core/state/benchmarks" pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1" + "github.com/prysmaticlabs/prysm/shared/benchutil" "github.com/prysmaticlabs/prysm/shared/interop" "github.com/prysmaticlabs/prysm/shared/params" "github.com/prysmaticlabs/prysm/shared/testutil" ) +var ( + outputDir = flag.String("output-dir", "", "Directory to write SSZ files to") + overwrite = flag.Bool("overwrite", false, "If SSZ files exist in the output directory, they will be overwritten") +) + func main() { + flag.Parse() + if *outputDir == "" { + log.Fatal("Please specify --output-dir to write SSZ files to") + } + + if !*overwrite { + if _, err := os.Stat(path.Join(*outputDir, benchutil.BState1EpochFileName)); err == nil { + log.Fatal("The file exists. Use a different file name or the --overwrite flag") + } + if _, err := os.Stat(path.Join(*outputDir, benchutil.BState2EpochFileName)); err == nil { + log.Fatal("The file exists. Use a different file name or the --overwrite flag") + } + if _, err := os.Stat(path.Join(*outputDir, benchutil.FullBlockFileName)); err == nil { + log.Fatal("The file exists. Use a different file name or the --overwrite flag") + } + } + + if err := os.MkdirAll(*outputDir, os.ModePerm); err != nil { + log.Fatal(err) + } + + log.Printf("Output dir is: %s", *outputDir) log.Println("Generating genesis state") // Generating this for the 2 following states. if err := generateGenesisBeaconState(); err != nil { - log.Fatal(err) + log.Fatalf("Could not generate genesis state: %v", err) } log.Println("Generating full block and state after 1 skipped epoch") if err := generateMarshalledFullStateAndBlock(); err != nil { - log.Fatal(err) + log.Fatalf("Could not generate full state and block: %v", err) } log.Println("Generating state after 2 fully attested epochs") if err := generate2FullEpochState(); err != nil { - log.Fatal(err) + log.Fatalf("Could not generate 2 full epoch state: %v", err) } - // Removing this since its 10MB large and no longer needed. - if err := os.Remove(bench.FilePath(bench.GenesisFileName)); err != nil { + // Removing the genesis state SSZ since its 10MB large and no longer needed. + if err := os.Remove(path.Join(*outputDir, benchutil.GenesisFileName)); err != nil { log.Fatal(err) } } func generateGenesisBeaconState() error { - genesisState, _, err := interop.GenerateGenesisState(0, bench.ValidatorCount) + genesisState, _, err := interop.GenerateGenesisState(0, benchutil.ValidatorCount) if err != nil { return err } @@ -46,20 +76,20 @@ func generateGenesisBeaconState() error { if err != nil { return err } - if err := ioutil.WriteFile(bench.FilePath(bench.GenesisFileName), beaconBytes, 0644); err != nil { + if err := ioutil.WriteFile(path.Join(*outputDir, benchutil.GenesisFileName), beaconBytes, 0644); err != nil { return err } return nil } func generateMarshalledFullStateAndBlock() error { - bench.SetConfig() + benchutil.SetBenchmarkConfig() beaconState, err := genesisBeaconState() if err != nil { return err } - privs, _, err := interop.DeterministicallyGenerateKeys(0, bench.ValidatorCount) + privs, _, err := interop.DeterministicallyGenerateKeys(0, benchutil.ValidatorCount) if err != nil { return err } @@ -78,7 +108,7 @@ func generateMarshalledFullStateAndBlock() error { } attConfig := &testutil.BlockGenConfig{ - NumAttestations: bench.AttestationsPerEpoch / slotsPerEpoch, + NumAttestations: benchutil.AttestationsPerEpoch / slotsPerEpoch, } atts := []*ethpb.Attestation{} @@ -92,18 +122,18 @@ func generateMarshalledFullStateAndBlock() error { block, err = testutil.GenerateFullBlock(beaconState, privs, attConfig, beaconState.Slot) if err != nil { - return err + return errors.Wrap(err, "could not generate full block") } block.Block.Body.Attestations = append(atts, block.Block.Body.Attestations...) s, err := state.CalculateStateRoot(context.Background(), beaconState, block) if err != nil { - return err + return errors.Wrap(err, "could not calculate state root") } block.Block.StateRoot = s[:] blockRoot, err := ssz.HashTreeRoot(block.Block) if err != nil { - return err + return errors.Wrap(err, "could not get signing root of block") } // Temporarily incrementing the beacon state slot here since BeaconProposerIndex is a // function deterministic on beacon state slot. @@ -120,7 +150,7 @@ func generateMarshalledFullStateAndBlock() error { if err != nil { return err } - if err := ioutil.WriteFile(bench.FilePath(bench.BState1EpochFileName), beaconBytes, 0644); err != nil { + if err := ioutil.WriteFile(path.Join(*outputDir, benchutil.BState1EpochFileName), beaconBytes, 0644); err != nil { return err } @@ -134,26 +164,26 @@ func generateMarshalledFullStateAndBlock() error { if err != nil { return err } - if err := ioutil.WriteFile(bench.FilePath(bench.FullBlockFileName), blockBytes, 0644); err != nil { + if err := ioutil.WriteFile(path.Join(*outputDir, benchutil.FullBlockFileName), blockBytes, 0644); err != nil { return err } return nil } func generate2FullEpochState() error { - bench.SetConfig() + benchutil.SetBenchmarkConfig() beaconState, err := genesisBeaconState() if err != nil { return err } - privs, _, err := interop.DeterministicallyGenerateKeys(0, bench.ValidatorCount) + privs, _, err := interop.DeterministicallyGenerateKeys(0, benchutil.ValidatorCount) if err != nil { return err } attConfig := &testutil.BlockGenConfig{ - NumAttestations: bench.AttestationsPerEpoch / params.BeaconConfig().SlotsPerEpoch, + NumAttestations: benchutil.AttestationsPerEpoch / params.BeaconConfig().SlotsPerEpoch, } for i := uint64(0); i < params.BeaconConfig().SlotsPerEpoch*2-1; i++ { @@ -171,20 +201,20 @@ func generate2FullEpochState() error { if err != nil { return err } - if err := ioutil.WriteFile(bench.FilePath(bench.BState2EpochFileName), beaconBytes, 0644); err != nil { + if err := ioutil.WriteFile(path.Join(*outputDir, benchutil.BState2EpochFileName), beaconBytes, 0644); err != nil { return err } return nil } func genesisBeaconState() (*pb.BeaconState, error) { - beaconBytes, err := ioutil.ReadFile(bench.FilePath(bench.GenesisFileName)) + beaconBytes, err := ioutil.ReadFile(path.Join(*outputDir, benchutil.GenesisFileName)) if err != nil { - return nil, err + return nil, errors.Wrap(err, "cannot read genesis state file") } genesisState := &pb.BeaconState{} if err := ssz.Unmarshal(beaconBytes, genesisState); err != nil { - return nil, err + return nil, errors.Wrap(err, "cannot unmarshal genesis state file") } return genesisState, nil }