Embedded mainnet genesis state + load genesis.ssz from file (#8614)

* Update rules_go and fix proto conflicts

* gaz

* Update generated code

* First pass inclusion of using baked states

* more emptypb fixes

* remove testnet genesis files, only embed mainnet

* Refactoring for SaveGenesisData, fix tests that use mainnet config but do not support mainnet genesis values

* a bit more refactoring, load genesis from a file. Needs tests still

* Add method to ensure an embedded genesis file also has the appropriate genesis block

* gofmt

* more clear error

* Check genesis fork version to ensure testnet config matches genesis file

* viz

* test for SaveGenesisData

* More genesis db method tests

* Merge

* Minor tweaks, lint, fmt, etc

* Add more test to genesis db methods

* Revert beacon-chain/state/stateV0/BUILD.bazel

* Update beacon-chain/db/iface/errors.go

Co-authored-by: Raul Jordan <raul@prysmaticlabs.com>

* PR feedback

* Update beacon-chain/db/kv/genesis.go

Co-authored-by: terence tsao <terence@prysmaticlabs.com>

* fmt.Errorf works better for nil errors

Co-authored-by: Raul Jordan <raul@prysmaticlabs.com>
Co-authored-by: terence tsao <terence@prysmaticlabs.com>

Author: 3 people

beacon-chain/blockchain/BUILD.bazel

@@ -27,7 +27,6 @@ go_library(
     deps = [
         "//beacon-chain/cache:go_default_library",
         "//beacon-chain/cache/depositcache:go_default_library",
-        "//beacon-chain/core/blocks:go_default_library",
         "//beacon-chain/core/epoch/precompute:go_default_library",
         "//beacon-chain/core/feed:go_default_library",
         "//beacon-chain/core/feed/state:go_default_library",
@@ -84,6 +83,7 @@ go_test(
         "checktags_test.go",
         "head_test.go",
         "info_test.go",
+        "init_test.go",
         "metrics_test.go",
         "process_attestation_test.go",
         "process_block_test.go",
@@ -130,6 +130,7 @@ go_test(
     srcs = [
         "chain_info_norace_test.go",
         "checktags_test.go",
+        "init_test.go",
         "receive_block_test.go",
         "service_norace_test.go",
     ],

beacon-chain/blockchain/init_test.go

@@ -0,0 +1,12 @@
+package blockchain
+
+import (
+	"github.com/prysmaticlabs/prysm/shared/params"
+)
+
+func init() {
+	// Override network name so that hardcoded genesis files are not loaded.
+	cfg := params.BeaconConfig()
+	cfg.ConfigName = "test"
+	params.OverrideBeaconConfig(cfg)
+}

beacon-chain/blockchain/service.go

@@ -15,7 +15,6 @@ import (
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache"
 	"github.com/prysmaticlabs/prysm/beacon-chain/cache/depositcache"
-	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
 	statefeed "github.com/prysmaticlabs/prysm/beacon-chain/core/feed/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/helpers"
@@ -32,7 +31,6 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stateV0"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	"github.com/prysmaticlabs/prysm/cmd/beacon-chain/flags"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
 	"github.com/prysmaticlabs/prysm/shared/params"
 	"github.com/prysmaticlabs/prysm/shared/slotutil"
@@ -336,38 +334,20 @@ func (s *Service) Status() error {
 
 // This gets called when beacon chain is first initialized to save genesis data (state, block, and more) in db.
 func (s *Service) saveGenesisData(ctx context.Context, genesisState iface.BeaconState) error {
-	stateRoot, err := genesisState.HashTreeRoot(ctx)
-	if err != nil {
-		return err
+	if err := s.cfg.BeaconDB.SaveGenesisData(ctx, genesisState); err != nil {
+		return errors.Wrap(err, "could not save genesis data")
+	}
+	genesisBlk, err := s.cfg.BeaconDB.GenesisBlock(ctx)
+	if err != nil || genesisBlk == nil {
+		return fmt.Errorf("could not load genesis block: %v", err)
 	}
-	genesisBlk := blocks.NewGenesisBlock(stateRoot[:])
 	genesisBlkRoot, err := genesisBlk.Block.HashTreeRoot()
 	if err != nil {
 		return errors.Wrap(err, "could not get genesis block root")
 	}
-	s.genesisRoot = genesisBlkRoot
-
-	if err := s.cfg.BeaconDB.SaveBlock(ctx, genesisBlk); err != nil {
-		return errors.Wrap(err, "could not save genesis block")
-	}
-	if err := s.cfg.BeaconDB.SaveState(ctx, genesisState, genesisBlkRoot); err != nil {
-		return errors.Wrap(err, "could not save genesis state")
-	}
-	if err := s.cfg.BeaconDB.SaveStateSummary(ctx, &pb.StateSummary{
-		Slot: 0,
-		Root: genesisBlkRoot[:],
-	}); err != nil {
-		return err
-	}
 
-	s.cfg.StateGen.SaveFinalizedState(0, genesisBlkRoot, genesisState)
-
-	if err := s.cfg.BeaconDB.SaveHeadBlockRoot(ctx, genesisBlkRoot); err != nil {
-		return errors.Wrap(err, "could not save head block root")
-	}
-	if err := s.cfg.BeaconDB.SaveGenesisBlockRoot(ctx, genesisBlkRoot); err != nil {
-		return errors.Wrap(err, "could not save genesis block root")
-	}
+	s.genesisRoot = genesisBlkRoot
+	s.cfg.StateGen.SaveFinalizedState(0 /*slot*/, genesisBlkRoot, genesisState)
 
 	// Finalized checkpoint at genesis is a zero hash.
 	genesisCheckpoint := genesisState.FinalizedCheckpoint()
@@ -392,7 +372,6 @@ func (s *Service) saveGenesisData(ctx context.Context, genesisState iface.Beacon
 	}
 
 	s.setHead(genesisBlkRoot, genesisBlk, genesisState)
-
 	return nil
 }
 

beacon-chain/db/alias.go

@@ -22,3 +22,7 @@ type HeadAccessDatabase = iface.HeadAccessDatabase
 // key-value or relational database in practice. This is the full database interface which should
 // not be used often. Prefer a more restrictive interface in this package.
 type Database = iface.Database
+
+// ErrExistingGenesisState is an error when the user attempts to save a different genesis state
+// when one already exists in a database.
+var ErrExistingGenesisState = iface.ErrExistingGenesisState

beacon-chain/db/iface/BUILD.bazel

@@ -2,7 +2,10 @@ load("@prysm//tools/go:def.bzl", "go_library")
 
 go_library(
     name = "go_default_library",
-    srcs = ["interface.go"],
+    srcs = [
+        "errors.go",
+        "interface.go",
+    ],
     importpath = "github.com/prysmaticlabs/prysm/beacon-chain/db/iface",
     # Other packages must use github.com/prysmaticlabs/prysm/beacon-chain/db.Database alias.
     visibility = ["//beacon-chain/db:__subpackages__"],

beacon-chain/db/iface/errors.go

@@ -0,0 +1,11 @@
+package iface
+
+import (
+	"errors"
+)
+
+var (
+	// ErrExistingGenesisState is an error when the user attempts to save a different genesis state
+	// when one already exists in a database.
+	ErrExistingGenesisState = errors.New("genesis state exists already in the DB")
+)

beacon-chain/db/iface/interface.go

@@ -99,6 +99,11 @@ type HeadAccessDatabase interface {
 	// Block related methods.
 	HeadBlock(ctx context.Context) (*eth.SignedBeaconBlock, error)
 	SaveHeadBlockRoot(ctx context.Context, blockRoot [32]byte) error
+
+	// Genesis operations.
+	LoadGenesis(ctx context.Context, r io.Reader) error
+	SaveGenesisData(ctx context.Context, state iface.BeaconState) error
+	EnsureEmbeddedGenesis(ctx context.Context) error
 }
 
 // Database interface with full access.

beacon-chain/db/kafka/passthrough.go

@@ -2,6 +2,7 @@ package kafka
 
 import (
 	"context"
+	"io"
 
 	"github.com/ethereum/go-ethereum/common"
 	types "github.com/prysmaticlabs/eth2-types"
@@ -266,3 +267,18 @@ func (e Exporter) RunMigrations(ctx context.Context) error {
 func (e Exporter) CleanUpDirtyStates(ctx context.Context, slotsPerArchivedPoint types.Slot) error {
 	return e.db.RunMigrations(ctx)
 }
+
+// LoadGenesisFromFile -- passthrough
+func (e Exporter) LoadGenesis(ctx context.Context, r io.Reader) error {
+	return e.db.LoadGenesis(ctx, r)
+}
+
+// SaveGenesisData -- passthrough
+func (e Exporter) SaveGenesisData(ctx context.Context, state iface.BeaconState) error {
+	return e.db.SaveGenesisData(ctx, state)
+}
+
+// EnsureEmbeddedGenesis -- passthrough.
+func (e Exporter) EnsureEmbeddedGenesis(ctx context.Context) error {
+	return e.db.EnsureEmbeddedGenesis(ctx)
+}

beacon-chain/db/kv/BUILD.bazel

@@ -11,6 +11,7 @@ go_library(
         "deposit_contract.go",
         "encoding.go",
         "finalized_block_roots.go",
+        "genesis.go",
         "kv.go",
         "log.go",
         "migration.go",
@@ -32,9 +33,11 @@ go_library(
         "//tools:__subpackages__",
     ],
     deps = [
+        "//beacon-chain/core/blocks:go_default_library",
         "//beacon-chain/core/helpers:go_default_library",
         "//beacon-chain/db/filters:go_default_library",
         "//beacon-chain/db/iface:go_default_library",
+        "//beacon-chain/state/genesis:go_default_library",
         "//beacon-chain/state/interface:go_default_library",
         "//beacon-chain/state/stateV0:go_default_library",
         "//proto/beacon/db:go_default_library",
@@ -70,6 +73,8 @@ go_test(
         "deposit_contract_test.go",
         "encoding_test.go",
         "finalized_block_roots_test.go",
+        "genesis_test.go",
+        "init_test.go",
         "kv_test.go",
         "migration_archived_index_test.go",
         "migration_block_slot_index_test.go",
@@ -80,9 +85,11 @@ go_test(
         "state_test.go",
         "utils_test.go",
     ],
+    data = glob(["testdata/**"]),
     embed = [":go_default_library"],
     deps = [
         "//beacon-chain/db/filters:go_default_library",
+        "//beacon-chain/db/iface:go_default_library",
         "//beacon-chain/state/interface:go_default_library",
         "//proto/beacon/db:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
@@ -97,6 +104,7 @@ go_test(
         "@com_github_prysmaticlabs_eth2_types//:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
         "@in_gopkg_d4l3k_messagediff_v1//:go_default_library",
+        "@io_bazel_rules_go//go/tools/bazel:go_default_library",
         "@io_etcd_go_bbolt//:go_default_library",
     ],
 )

beacon-chain/db/kv/genesis.go

@@ -0,0 +1,115 @@
+package kv
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+
+	"github.com/pkg/errors"
+	"github.com/prysmaticlabs/prysm/beacon-chain/core/blocks"
+	dbIface "github.com/prysmaticlabs/prysm/beacon-chain/db/iface"
+	iface "github.com/prysmaticlabs/prysm/beacon-chain/state/interface"
+	state "github.com/prysmaticlabs/prysm/beacon-chain/state/stateV0"
+	pbp2p "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/params"
+)
+
+// SaveGenesisData bootstraps the beaconDB with a given genesis state.
+func (s *Store) SaveGenesisData(ctx context.Context, genesisState iface.BeaconState) error {
+	stateRoot, err := genesisState.HashTreeRoot(ctx)
+	if err != nil {
+		return err
+	}
+	genesisBlk := blocks.NewGenesisBlock(stateRoot[:])
+	genesisBlkRoot, err := genesisBlk.Block.HashTreeRoot()
+	if err != nil {
+		return errors.Wrap(err, "could not get genesis block root")
+	}
+	if err := s.SaveBlock(ctx, genesisBlk); err != nil {
+		return errors.Wrap(err, "could not save genesis block")
+	}
+	if err := s.SaveState(ctx, genesisState, genesisBlkRoot); err != nil {
+		return errors.Wrap(err, "could not save genesis state")
+	}
+	if err := s.SaveStateSummary(ctx, &pbp2p.StateSummary{
+		Slot: 0,
+		Root: genesisBlkRoot[:],
+	}); err != nil {
+		return err
+	}
+
+	if err := s.SaveHeadBlockRoot(ctx, genesisBlkRoot); err != nil {
+		return errors.Wrap(err, "could not save head block root")
+	}
+	if err := s.SaveGenesisBlockRoot(ctx, genesisBlkRoot); err != nil {
+		return errors.Wrap(err, "could not save genesis block root")
+	}
+
+	return nil
+}
+
+// LoadGenesisFromFile loads a genesis state from a given file path, if no genesis exists already.
+func (s *Store) LoadGenesis(ctx context.Context, r io.Reader) error {
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		return err
+	}
+	st := &pbp2p.BeaconState{}
+	if err := st.UnmarshalSSZ(b); err != nil {
+		return err
+	}
+	gs, err := state.InitializeFromProtoUnsafe(st)
+	if err != nil {
+		return err
+	}
+	existing, err := s.GenesisState(ctx)
+	if err != nil {
+		return err
+	}
+	// If some different genesis state existed already, return an error. The same genesis state is
+	// considered a no-op.
+	if existing != nil {
+		a, err := existing.HashTreeRoot(ctx)
+		if err != nil {
+			return err
+		}
+		b, err := gs.HashTreeRoot(ctx)
+		if err != nil {
+			return err
+		}
+		if a == b {
+			return nil
+		}
+		return dbIface.ErrExistingGenesisState
+	}
+
+	if !bytes.Equal(gs.Fork().CurrentVersion, params.BeaconConfig().GenesisForkVersion) {
+		return fmt.Errorf("loaded genesis fork version (%#x) does not match config genesis "+
+			"fork version (%#x)", gs.Fork().CurrentVersion, params.BeaconConfig().GenesisForkVersion)
+	}
+
+	return s.SaveGenesisData(ctx, gs)
+}
+
+// EnsureEmbeddedGenesis checks that a genesis block has been generated when an embedded genesis
+// state is used. If a genesis block does not exist, but a genesis state does, then we should call
+// SaveGenesisData on the existing genesis state.
+func (s *Store) EnsureEmbeddedGenesis(ctx context.Context) error {
+	gb, err := s.GenesisBlock(ctx)
+	if err != nil {
+		return err
+	}
+	if gb != nil {
+		return nil
+	}
+	gs, err := s.GenesisState(ctx)
+	if err != nil {
+		return err
+	}
+	if gs != nil {
+		return s.SaveGenesisData(ctx, gs)
+	}
+	return nil
+}