Merge pull request #649 from ehnuje/0824-chaindata-test

cmd, blockchain: added overwrite-genesis flag

blockchain/blockchain.go

@@ -23,7 +23,16 @@ package blockchain
 import (
 	"errors"
 	"fmt"
-	"github.com/hashicorp/golang-lru"
+	"io"
+	"math/big"
+	mrand "math/rand"
+	"reflect"
+	"strconv"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	lru "github.com/hashicorp/golang-lru"
 	"github.com/klaytn/klaytn/blockchain/state"
 	"github.com/klaytn/klaytn/blockchain/types"
 	"github.com/klaytn/klaytn/blockchain/vm"
@@ -39,14 +48,6 @@ import (
 	"github.com/klaytn/klaytn/storage/statedb"
 	"github.com/rcrowley/go-metrics"
 	"gopkg.in/karalabe/cookiejar.v2/collections/prque"
-	"io"
-	"math/big"
-	mrand "math/rand"
-	"reflect"
-	"strconv"
-	"sync"
-	"sync/atomic"
-	"time"
 )
 
 var (

blockchain/genesis.go

@@ -26,6 +26,9 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"math/big"
+	"strings"
+
 	"github.com/klaytn/klaytn/blockchain/state"
 	"github.com/klaytn/klaytn/blockchain/types"
 	"github.com/klaytn/klaytn/common"
@@ -34,8 +37,6 @@ import (
 	"github.com/klaytn/klaytn/params"
 	"github.com/klaytn/klaytn/ser/rlp"
 	"github.com/klaytn/klaytn/storage/database"
-	"math/big"
-	"strings"
 )
 
 //go:generate gencodec -type Genesis -field-override genesisSpecMarshaling -out gen_genesis.go
@@ -136,6 +137,31 @@ func (e *GenesisMismatchError) Error() string {
 	return fmt.Sprintf("database already contains an incompatible genesis block (have %x, new %x)", e.Stored[:8], e.New[:8])
 }
 
+// findBlockWithState returns the latest block with state.
+func findBlockWithState(db database.DBManager) *types.Block {
+	headBlock := db.ReadBlockByHash(db.ReadHeadBlockHash())
+	if headBlock == nil {
+		logger.Crit("failed to read head block by head block hash")
+	}
+
+	startBlock := headBlock
+	for _, err := state.New(headBlock.Root(), state.NewDatabase(db)); err != nil; {
+		if headBlock.NumberU64() == 0 {
+			logger.Crit("failed to find state from the head block to the genesis block",
+				"headBlockNum", headBlock.NumberU64(),
+				"headBlockHash", headBlock.Hash().String(), "headBlockRoot", headBlock.Root().String())
+		}
+		headBlock = db.ReadBlockByNumber(headBlock.NumberU64() - 1)
+		if headBlock == nil {
+			logger.Crit("failed to read previous block by head block number")
+		}
+		logger.Warn("found previous block", "blockNum", headBlock.NumberU64())
+	}
+	logger.Info("found the latest block with state",
+		"blockNum", headBlock.NumberU64(), "startedNum", startBlock.NumberU64())
+	return headBlock
+}
+
 // SetupGenesisBlock writes or updates the genesis block in db.
 // The block that will be used is:
 //
@@ -149,7 +175,7 @@ func (e *GenesisMismatchError) Error() string {
 // error is a *params.ConfigCompatError and the new, unwritten config is returned.
 //
 // The returned chain configuration is never nil.
-func SetupGenesisBlock(db database.DBManager, genesis *Genesis, networkId uint64, isPrivate bool) (*params.ChainConfig, common.Hash, error) {
+func SetupGenesisBlock(db database.DBManager, genesis *Genesis, networkId uint64, isPrivate, overwriteGenesis bool) (*params.ChainConfig, common.Hash, error) {
 	if genesis != nil && genesis.Config == nil {
 		return params.AllGxhashProtocolChanges, common.Hash{}, errGenesisNoConfig
 	}
@@ -179,13 +205,23 @@ func SetupGenesisBlock(db database.DBManager, genesis *Genesis, networkId uint64
 		}
 		// Initialize DeriveSha implementation
 		InitDeriveSha(genesis.Config.DeriveShaImpl)
-		block, err := genesis.Commit(db)
+		block, err := genesis.Commit(common.Hash{}, db)
 		return genesis.Config, block.Hash(), err
 	}
 
 	// Check whether the genesis block is already written.
 	if genesis != nil {
-		hash := genesis.ToBlock(nil).Hash()
+		// If overwriteGenesis is true, overwrite existing genesis block with the new one.
+		// This is to run a test with pre-existing data.
+		if overwriteGenesis {
+			headBlock := findBlockWithState(db)
+			logger.Warn("Trying to overwrite original genesis block with the new one",
+				"headBlockHash", headBlock.Hash().String(), "headBlockNum", headBlock.NumberU64())
+			newGenesisBlock, err := genesis.Commit(headBlock.Root(), db)
+			return genesis.Config, newGenesisBlock.Hash(), err
+		}
+		// This is the usual path which does not overwrite genesis block with the new one.
+		hash := genesis.ToBlock(common.Hash{}, nil).Hash()
 		if hash != stored {
 			return genesis.Config, hash, &GenesisMismatchError{stored, hash}
 		}
@@ -241,11 +277,11 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig {
 
 // ToBlock creates the genesis block and writes state of a genesis specification
 // to the given database (or discards it if nil).
-func (g *Genesis) ToBlock(db database.DBManager) *types.Block {
+func (g *Genesis) ToBlock(baseStateRoot common.Hash, db database.DBManager) *types.Block {
 	if db == nil {
 		db = database.NewMemoryDBManager()
 	}
-	stateDB, _ := state.New(common.Hash{}, state.NewDatabase(db))
+	stateDB, _ := state.New(baseStateRoot, state.NewDatabase(db))
 	for addr, account := range g.Alloc {
 		if len(account.Code) != 0 {
 			stateDB.SetCode(addr, account.Code)
@@ -279,8 +315,8 @@ func (g *Genesis) ToBlock(db database.DBManager) *types.Block {
 
 // Commit writes the block and state of a genesis specification to the database.
 // The block is committed as the canonical head block.
-func (g *Genesis) Commit(db database.DBManager) (*types.Block, error) {
-	block := g.ToBlock(db)
+func (g *Genesis) Commit(baseStateRoot common.Hash, db database.DBManager) (*types.Block, error) {
+	block := g.ToBlock(baseStateRoot, db)
 	if block.Number().Sign() != 0 {
 		return nil, fmt.Errorf("can't commit genesis block with number > 0")
 	}
@@ -308,7 +344,7 @@ func (g *Genesis) MustCommit(db database.DBManager) *types.Block {
 	}
 	InitDeriveSha(config.DeriveShaImpl)
 
-	block, err := g.Commit(db)
+	block, err := g.Commit(common.Hash{}, db)
 	if err != nil {
 		panic(err)
 	}

blockchain/genesis_test.go

@@ -68,7 +68,7 @@ func TestSetupGenesis(t *testing.T) {
 		{
 			name: "genesis without ChainConfig",
 			fn: func(db database.DBManager) (*params.ChainConfig, common.Hash, error) {
-				return SetupGenesisBlock(db, new(Genesis), params.UnusedNetworkId, false)
+				return SetupGenesisBlock(db, new(Genesis), params.UnusedNetworkId, false, false)
 			},
 			wantErr:    errGenesisNoConfig,
 			wantConfig: params.AllGxhashProtocolChanges,

cmd/kcn/main.go

@@ -59,6 +59,7 @@ var cnHelpFlagGroups = []utils.FlagGroup{
 			utils.SrvTypeFlag,
 			utils.ExtraDataFlag,
 			nodecmd.ConfigFileFlag,
+			utils.OverwriteGenesisFlag,
 		},
 	},
 	{

cmd/ken/main.go

@@ -59,6 +59,7 @@ var enHelpFlagGroups = []utils.FlagGroup{
 			utils.SrvTypeFlag,
 			utils.ExtraDataFlag,
 			nodecmd.ConfigFileFlag,
+			utils.OverwriteGenesisFlag,
 		},
 	},
 	{

cmd/kpn/main.go

@@ -62,6 +62,7 @@ var pnHelpFlagGroups = []utils.FlagGroup{
 			utils.SrvTypeFlag,
 			utils.ExtraDataFlag,
 			nodecmd.ConfigFileFlag,
+			utils.OverwriteGenesisFlag,
 		},
 	},
 	{

cmd/kscn/main.go

@@ -59,6 +59,7 @@ var scnHelpFlagGroups = []utils.FlagGroup{
 			utils.SrvTypeFlag,
 			utils.ExtraDataFlag,
 			nodecmd.ConfigFileFlag,
+			utils.OverwriteGenesisFlag,
 		},
 	},
 	{

cmd/ksen/main.go

@@ -59,6 +59,7 @@ var senHelpFlagGroups = []utils.FlagGroup{
 			utils.SrvTypeFlag,
 			utils.ExtraDataFlag,
 			nodecmd.ConfigFileFlag,
+			utils.OverwriteGenesisFlag,
 		},
 	},
 	{

cmd/kspn/main.go

@@ -62,6 +62,7 @@ var spnHelpFlagGroups = []utils.FlagGroup{
 			utils.SrvTypeFlag,
 			utils.ExtraDataFlag,
 			nodecmd.ConfigFileFlag,
+			utils.OverwriteGenesisFlag,
 		},
 	},
 	{

cmd/utils/flags.go

@@ -129,6 +129,10 @@ var (
 		Name:  "lightkdf",
 		Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength",
 	}
+	OverwriteGenesisFlag = cli.BoolFlag{
+		Name:  "overwrite-genesis",
+		Usage: "Overwrites genesis block with the given new genesis block for testing purpose",
+	}
 	// Transaction pool settings
 	TxPoolNoLocalsFlag = cli.BoolFlag{
 		Name:  "txpool.nolocals",
@@ -1268,6 +1272,8 @@ func SetKlayConfig(ctx *cli.Context, stack *node.Node, cfg *cn.Config) {
 		log.Fatalf("%v should be power of 2 but %v is not!", NumStateTrieShardsFlag.Name, cfg.NumStateTrieShards)
 	}
 
+	cfg.OverwriteGenesis = ctx.GlobalBool(OverwriteGenesisFlag.Name)
+
 	cfg.LevelDBCompression = database.LevelDBCompressionType(ctx.GlobalInt(LevelDBCompressionTypeFlag.Name))
 	cfg.LevelDBBufferPool = !ctx.GlobalIsSet(LevelDBNoBufferPoolFlag.Name)
 	cfg.LevelDBCacheSize = ctx.GlobalInt(LevelDBCacheSizeFlag.Name)

cmd/utils/nodecmd/chaincmd.go

@@ -55,6 +55,7 @@ var (
 			utils.DynamoDBWriteCapacityFlag,
 			utils.LevelDBCompressionTypeFlag,
 			utils.DataDirFlag,
+			utils.OverwriteGenesisFlag,
 		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
@@ -138,6 +139,7 @@ func initGenesis(ctx *cli.Context) error {
 	parallelDBWrite := !ctx.GlobalIsSet(utils.NoParallelDBWriteFlag.Name)
 	singleDB := ctx.GlobalIsSet(utils.SingleDBFlag.Name)
 	numStateTrieShards := ctx.GlobalUint(utils.NumStateTrieShardsFlag.Name)
+	overwriteGenesis := ctx.GlobalBool(utils.OverwriteGenesisFlag.Name)
 	for _, name := range []string{"chaindata", "lightchaindata"} {
 		dbc := &database.DBConfig{Dir: name, DBType: database.LevelDB, ParallelDBWrite: parallelDBWrite,
 			SingleDB: singleDB, NumStateTrieShards: numStateTrieShards,
@@ -146,7 +148,7 @@ func initGenesis(ctx *cli.Context) error {
 		// Initialize DeriveSha implementation
 		blockchain.InitDeriveSha(genesis.Config.DeriveShaImpl)
 
-		_, hash, err := blockchain.SetupGenesisBlock(chaindb, genesis, params.UnusedNetworkId, false)
+		_, hash, err := blockchain.SetupGenesisBlock(chaindb, genesis, params.UnusedNetworkId, false, overwriteGenesis)
 		if err != nil {
 			log.Fatalf("Failed to write genesis block: %v", err)
 		}

cmd/utils/nodecmd/nodeflags.go

@@ -33,6 +33,7 @@ var CommonNodeFlags = []cli.Flag{
 	utils.PasswordFileFlag,
 	utils.DbTypeFlag,
 	utils.DataDirFlag,
+	utils.OverwriteGenesisFlag,
 	utils.KeyStoreDirFlag,
 	utils.TxPoolNoLocalsFlag,
 	utils.TxPoolAllowLocalAnchorTxFlag,

consensus/clique/snapshot_test.go

@@ -19,6 +19,9 @@ package clique
 import (
 	"bytes"
 	"crypto/ecdsa"
+	"sort"
+	"testing"
+
 	"github.com/klaytn/klaytn/blockchain"
 	"github.com/klaytn/klaytn/blockchain/types"
 	"github.com/klaytn/klaytn/blockchain/vm"
@@ -28,8 +31,6 @@ import (
 	"github.com/klaytn/klaytn/params"
 	"github.com/klaytn/klaytn/ser/rlp"
 	"github.com/klaytn/klaytn/storage/database"
-	"sort"
-	"testing"
 )
 
 // testerAccountPool is a pool to maintain currently active tester accounts,
@@ -403,7 +404,7 @@ func TestClique(t *testing.T) {
 		}
 		// Create a pristine blockchain with the genesis injected
 		db := database.NewMemoryDBManager()
-		genesis.Commit(db)
+		genesis.Commit(common.Hash{}, db)
 
 		// Assemble a chain of headers from the cast votes
 		config := *params.TestChainConfig
@@ -414,7 +415,7 @@ func TestClique(t *testing.T) {
 		engine := New(config.Clique, db)
 		engine.fakeBlockScore = true
 
-		blocks, _ := blockchain.GenerateChain(&config, genesis.ToBlock(db), engine, db, len(tt.votes), func(j int, gen *blockchain.BlockGen) {
+		blocks, _ := blockchain.GenerateChain(&config, genesis.ToBlock(common.Hash{}, db), engine, db, len(tt.votes), func(j int, gen *blockchain.BlockGen) {
 			vote := new(governance.GovernanceVote)
 			if tt.votes[j].auth {
 				vote.Key = "addvalidator"

governance/default_test.go

@@ -17,6 +17,10 @@
 package governance
 
 import (
+	"math/big"
+	"reflect"
+	"testing"
+
 	"github.com/klaytn/klaytn/blockchain"
 	"github.com/klaytn/klaytn/blockchain/types"
 	"github.com/klaytn/klaytn/common"
@@ -26,9 +30,6 @@ import (
 	"github.com/klaytn/klaytn/ser/rlp"
 	"github.com/klaytn/klaytn/storage/database"
 	"github.com/stretchr/testify/assert"
-	"math/big"
-	"reflect"
-	"testing"
 )
 
 type voteValue struct {
@@ -528,7 +529,7 @@ func TestBaoBabGenesisHash(t *testing.T) {
 	blockchain.InitDeriveSha(genesis.Config.DeriveShaImpl)
 
 	db := database.NewMemoryDBManager()
-	block, _ := genesis.Commit(db)
+	block, _ := genesis.Commit(common.Hash{}, db)
 	if block.Hash() != baobabHash {
 		t.Errorf("Generated hash is not equal to Baobab's hash. Want %v, Have %v", baobabHash.String(), block.Hash().String())
 	}
@@ -541,7 +542,7 @@ func TestCypressGenesisHash(t *testing.T) {
 	blockchain.InitDeriveSha(genesis.Config.DeriveShaImpl)
 
 	db := database.NewMemoryDBManager()
-	block, _ := genesis.Commit(db)
+	block, _ := genesis.Commit(common.Hash{}, db)
 	if block.Hash() != cypressHash {
 		t.Errorf("Generated hash is not equal to Cypress's hash. Want %v, Have %v", cypressHash.String(), block.Hash().String())
 	}

node/cn/backend.go

@@ -204,7 +204,7 @@ func New(ctx *node.ServiceContext, config *Config) (*CN, error) {
 
 	chainDB := CreateDB(ctx, config, "chaindata")
 
-	chainConfig, genesisHash, genesisErr := blockchain.SetupGenesisBlock(chainDB, config.Genesis, config.NetworkId, config.IsPrivate)
+	chainConfig, genesisHash, genesisErr := blockchain.SetupGenesisBlock(chainDB, config.Genesis, config.NetworkId, config.IsPrivate, false)
 	if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok {
 		return nil, genesisErr
 	}

node/cn/config.go

@@ -93,6 +93,8 @@ type Config struct {
 	//LightServ  int `toml:",omitempty"` // Maximum percentage of time allowed for serving LES requests
 	//LightPeers int `toml:",omitempty"` // Maximum number of LES client peers
 
+	OverwriteGenesis bool
+
 	// Database options
 	DBType               database.DBType
 	SkipBcVersionCheck   bool `toml:"-"`

node/sc/mainbridge_test.go

@@ -18,6 +18,12 @@ package sc
 
 import (
 	"fmt"
+	"math/big"
+	"path"
+	"reflect"
+	"strings"
+	"testing"
+
 	"github.com/golang/mock/gomock"
 	"github.com/klaytn/klaytn/accounts"
 	"github.com/klaytn/klaytn/api"
@@ -37,11 +43,6 @@ import (
 	"github.com/klaytn/klaytn/params"
 	"github.com/klaytn/klaytn/storage/database"
 	"github.com/stretchr/testify/assert"
-	"math/big"
-	"path"
-	"reflect"
-	"strings"
-	"testing"
 )
 
 const testNetVersion = uint64(8888)
@@ -87,7 +88,7 @@ func testBlockChain(t *testing.T) *blockchain.BlockChain {
 	genesis.Config.Istanbul = governance.GetDefaultIstanbulConfig()
 	genesis.Config.UnitPrice = 25 * params.Ston
 
-	chainConfig, _, err := blockchain.SetupGenesisBlock(db, genesis, params.UnusedNetworkId, false)
+	chainConfig, _, err := blockchain.SetupGenesisBlock(db, genesis, params.UnusedNetworkId, false, false)
 	if _, ok := err.(*params.ConfigCompatError); err != nil && !ok {
 		t.Fatal(err)
 	}

tests/block_test_util.go

@@ -96,7 +96,7 @@ func (t *BlockTest) Run() error {
 
 	// import pre accounts & construct test genesis block & state root
 	db := database.NewMemoryDBManager()
-	gblock, err := t.genesis(config).Commit(db)
+	gblock, err := t.genesis(config).Commit(common.Hash{}, db)
 	if err != nil {
 		return err
 	}