Hardfork: execute pending txs in correct order

update/common.go

@@ -0,0 +1,114 @@
+package update
+
+import (
+	"github.com/ElrondNetwork/elrond-go/data"
+	"github.com/ElrondNetwork/elrond-go/data/block"
+)
+
+// TxInfo defines the structure which hold the tx info
+type TxInfo struct {
+	MbHash []byte
+	TxHash []byte
+	Tx     data.TransactionHandler
+}
+
+// GetPendingMiniBlocks get all the pending miniBlocks from epoch start metaBlock and unFinished metaBlocks
+func GetPendingMiniBlocks(
+	epochStartMetaBlock *block.MetaBlock,
+	unFinishedMetaBlocksMap map[string]*block.MetaBlock,
+) ([]block.MiniBlockHeader, error) {
+
+	if epochStartMetaBlock == nil {
+		return nil, ErrNilEpochStartMetaBlock
+	}
+	if unFinishedMetaBlocksMap == nil {
+		return nil, ErrNilUnFinishedMetaBlocksMap
+	}
+
+	pendingMiniBlocks := make([]block.MiniBlockHeader, 0)
+	nonceToHashMap := createNonceToHashMap(unFinishedMetaBlocksMap)
+
+	for _, shardData := range epochStartMetaBlock.EpochStart.LastFinalizedHeaders {
+		computedPendingMiniBlocks, err := computePendingMiniBlocksFromUnFinishedMetaBlocks(
+			shardData,
+			unFinishedMetaBlocksMap,
+			nonceToHashMap,
+			epochStartMetaBlock.GetNonce(),
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		pendingMiniBlocks = append(pendingMiniBlocks, computedPendingMiniBlocks...)
+	}
+
+	return pendingMiniBlocks, nil
+}
+
+// createNonceToHashMap creates a map of nonce to hash from all the given metaBlocks
+func createNonceToHashMap(unFinishedMetaBlocks map[string]*block.MetaBlock) map[uint64]string {
+	nonceToHashMap := make(map[uint64]string, len(unFinishedMetaBlocks))
+	for metaBlockHash, metaBlock := range unFinishedMetaBlocks {
+		nonceToHashMap[metaBlock.GetNonce()] = metaBlockHash
+	}
+
+	return nonceToHashMap
+}
+
+// computePendingMiniBlocksFromUnFinishedMetaBlocks computes all the pending miniBlocks from unFinished metaBlocks
+func computePendingMiniBlocksFromUnFinishedMetaBlocks(
+	epochStartShardData block.EpochStartShardData,
+	unFinishedMetaBlocks map[string]*block.MetaBlock,
+	nonceToHashMap map[uint64]string,
+	epochStartMetaBlockNonce uint64,
+) ([]block.MiniBlockHeader, error) {
+	pendingMiniBlocks := make([]block.MiniBlockHeader, 0)
+	pendingMiniBlocks = append(pendingMiniBlocks, epochStartShardData.PendingMiniBlockHeaders...)
+
+	firstPendingMetaBlock, ok := unFinishedMetaBlocks[string(epochStartShardData.FirstPendingMetaBlock)]
+	if !ok {
+		return nil, ErrWrongUnFinishedMetaHdrsMap
+	}
+
+	firstUnFinishedMetaBlockNonce := firstPendingMetaBlock.GetNonce()
+	for nonce := firstUnFinishedMetaBlockNonce + 1; nonce <= epochStartMetaBlockNonce; nonce++ {
+		metaBlockHash, exists := nonceToHashMap[nonce]
+		if !exists {
+			return nil, ErrWrongUnFinishedMetaHdrsMap
+		}
+
+		metaBlock, exists := unFinishedMetaBlocks[metaBlockHash]
+		if !exists {
+			return nil, ErrWrongUnFinishedMetaHdrsMap
+		}
+
+		pendingMiniBlocksFromMetaBlock := getAllMiniBlocksWithDst(metaBlock, epochStartShardData.ShardID)
+		pendingMiniBlocks = append(pendingMiniBlocks, pendingMiniBlocksFromMetaBlock...)
+	}
+
+	return pendingMiniBlocks, nil
+}
+
+// getAllMiniBlocksWithDst returns all miniBlock headers with the given destination from the given metaBlock
+func getAllMiniBlocksWithDst(metaBlock *block.MetaBlock, destShardID uint32) []block.MiniBlockHeader {
+	mbHdrs := make([]block.MiniBlockHeader, 0)
+	for i := 0; i < len(metaBlock.ShardInfo); i++ {
+		if metaBlock.ShardInfo[i].ShardID == destShardID {
+			continue
+		}
+
+		for _, mbHdr := range metaBlock.ShardInfo[i].ShardMiniBlockHeaders {
+			if mbHdr.ReceiverShardID == destShardID && mbHdr.SenderShardID != destShardID {
+				mbHdrs = append(mbHdrs, mbHdr)
+			}
+		}
+	}
+
+	for _, mbHdr := range metaBlock.MiniBlockHeaders {
+		if mbHdr.ReceiverShardID == destShardID && mbHdr.SenderShardID != destShardID {
+			mbHdrs = append(mbHdrs, mbHdr)
+		}
+	}
+
+	return mbHdrs
+}

update/errors.go

@@ -14,9 +14,6 @@ var ErrInvalidFolderName = errors.New("invalid folder name")
 // ErrNilStorage signals that storage is nil
 var ErrNilStorage = errors.New("nil storage")
 
-// ErrNilDataTrieContainer signals that data trie container is nil
-var ErrNilDataTrieContainer = errors.New("nil data trie container")
-
 // ErrNotSynced signals that syncing has not been finished yet
 var ErrNotSynced = errors.New("not synced")
 
@@ -95,8 +92,8 @@ var ErrNilMiniBlocksSyncHandler = errors.New("nil miniblocks sync handler")
 // ErrNilTransactionsSyncHandler signals that nil transactions sync handler was provided
 var ErrNilTransactionsSyncHandler = errors.New("nil transaction sync handler")
 
-// ErrWrongUnfinishedMetaHdrsMap signals that wrong unfinished meta headers map was provided
-var ErrWrongUnfinishedMetaHdrsMap = errors.New("wrong unfinished meta headers map")
+// ErrWrongUnFinishedMetaHdrsMap signals that wrong unFinished meta headers map was provided
+var ErrWrongUnFinishedMetaHdrsMap = errors.New("wrong unFinished meta headers map")
 
 // ErrNilAccounts signals that nil accounts was provided
 var ErrNilAccounts = errors.New("nil accounts")
@@ -221,8 +218,8 @@ var ErrNilTimeCache = errors.New("nil time cache")
 // ErrNilHardforkStorer signals that a nil hardfork storer has been provided
 var ErrNilHardforkStorer = errors.New("nil hardfork storer")
 
-// ErrExpectedOneMetablock signals that exactly one metablock should have been used
-var ErrExpectedOneMetablock = errors.New("expected one metablock")
+// ErrExpectedOneStartOfEpochMetaBlock signals that exactly one start of epoch metaBlock should have been used
+var ErrExpectedOneStartOfEpochMetaBlock = errors.New("expected one start of epoch metaBlock")
 
 // ErrImportingData signals that an import error occurred
 var ErrImportingData = errors.New("error importing data")
@@ -238,3 +235,24 @@ var ErrEmptyExportFolderPath = errors.New("empty export folder path")
 
 // ErrNilGenesisNodesSetupHandler signals that a nil genesis nodes setup handler has been provided
 var ErrNilGenesisNodesSetupHandler = errors.New("nil genesis nodes setup handler")
+
+// ErrWrongImportedMiniBlocksMap signals that wrong imported miniBlocks map was provided
+var ErrWrongImportedMiniBlocksMap = errors.New("wrong imported miniBlocks map was provided")
+
+// ErrWrongImportedTransactionsMap signals that wrong imported transactions map was provided
+var ErrWrongImportedTransactionsMap = errors.New("wrong imported transactions map was provided")
+
+// ErrMiniBlockNotFoundInImportedMap signals that the given miniBlock was not found in imported map
+var ErrMiniBlockNotFoundInImportedMap = errors.New("miniBlock was not found in imported map")
+
+// ErrTransactionNotFoundInImportedMap signals that the given transaction was not found in imported map
+var ErrTransactionNotFoundInImportedMap = errors.New("transaction was not found in imported map")
+
+// ErrNilEpochStartMetaBlock signals that a nil epoch start metaBlock was provided
+var ErrNilEpochStartMetaBlock = errors.New("nil epoch start metaBlock was provided")
+
+//ErrNilUnFinishedMetaBlocksMap signals that a nil unFinished metaBlocks map was provided
+var ErrNilUnFinishedMetaBlocksMap = errors.New("nil unFinished metaBlocks map was provided")
+
+//ErrDuplicatedMiniBlocksFound signals that duplicated miniBlocks were found
+var ErrDuplicatedMiniBlocksFound = errors.New("duplicated miniBlocks were found")

update/genesis/base.go

@@ -15,8 +15,11 @@ import (
 	"github.com/ElrondNetwork/elrond-go/update"
 )
 
-// MetaBlockIdentifier is the constant which defines the export/import identifier for metaBlock
-const MetaBlockIdentifier = "metaBlock"
+// EpochStartMetaBlockIdentifier is the constant which defines the export/import identifier for epoch start metaBlock
+const EpochStartMetaBlockIdentifier = "epochStartMetaBlock"
+
+// UnFinishedMetaBlocksIdentifier is the constant which defines the export/import identifier for unFinished metaBlocks
+const UnFinishedMetaBlocksIdentifier = "unFinishedMetaBlocks"
 
 // TransactionsIdentifier is the constant which defines the export/import identifier for transactions
 const TransactionsIdentifier = "transactions"

update/genesis/export.go

@@ -9,7 +9,7 @@ import (
 	"sort"
 	"strings"
 
-	logger "github.com/ElrondNetwork/elrond-go-logger"
+	"github.com/ElrondNetwork/elrond-go-logger"
 	"github.com/ElrondNetwork/elrond-go/core"
 	"github.com/ElrondNetwork/elrond-go/core/check"
 	"github.com/ElrondNetwork/elrond-go/data"
@@ -107,7 +107,12 @@ func (se *stateExport) ExportAll(epoch uint32) error {
 		log.LogIfError(errClose)
 	}()
 
-	err = se.exportMeta()
+	err = se.exportEpochStartMetaBlock()
+	if err != nil {
+		return err
+	}
+
+	err = se.exportUnFinishedMetaBlocks()
 	if err != nil {
 		return err
 	}
@@ -136,7 +141,7 @@ func (se *stateExport) exportAllTransactions() error {
 		return err
 	}
 
-	log.Debug("Exported transactions", "len", len(toExportTransactions))
+	log.Debug("Starting export for transactions", "len", len(toExportTransactions))
 	for key, tx := range toExportTransactions {
 		errExport := se.exportTx(key, tx)
 		if errExport != nil {
@@ -153,7 +158,7 @@ func (se *stateExport) exportAllMiniBlocks() error {
 		return err
 	}
 
-	log.Debug("Exported miniBlocks", "len", len(toExportMBs))
+	log.Debug("Starting export for miniBlocks", "len", len(toExportMBs))
 	for key, mb := range toExportMBs {
 		errExport := se.exportMBs(key, mb)
 		if errExport != nil {
@@ -170,6 +175,7 @@ func (se *stateExport) exportAllTries() error {
 		return err
 	}
 
+	log.Debug("Starting export for tries", "len", len(toExportTries))
 	for key, trie := range toExportTries {
 		err = se.exportTrie(key, trie)
 		if err != nil {
@@ -180,31 +186,71 @@ func (se *stateExport) exportAllTries() error {
 	return nil
 }
 
-func (se *stateExport) exportMeta() error {
+func (se *stateExport) exportEpochStartMetaBlock() error {
 	metaBlock, err := se.stateSyncer.GetEpochStartMetaBlock()
 	if err != nil {
 		return err
 	}
 
-	jsonData, err := json.Marshal(metaBlock)
+	log.Debug("Starting export for epoch start metaBlock")
+	err = se.exportMetaBlock(metaBlock, EpochStartMetaBlockIdentifier)
 	if err != nil {
 		return err
 	}
 
-	metaHash := se.hasher.Compute(string(jsonData))
-	versionKey := CreateVersionKey(metaBlock, metaHash)
+	err = se.hardforkStorer.FinishedIdentifier(EpochStartMetaBlockIdentifier)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (se *stateExport) exportUnFinishedMetaBlocks() error {
+	unFinishedMetaBlocks, err := se.stateSyncer.GetUnFinishedMetaBlocks()
+	if err != nil {
+		return err
+	}
 
-	err = se.hardforkStorer.Write(MetaBlockIdentifier, []byte(versionKey), jsonData)
+	log.Debug("Starting export for unFinished metaBlocks", "len", len(unFinishedMetaBlocks))
+	for _, metaBlock := range unFinishedMetaBlocks {
+		err := se.exportMetaBlock(metaBlock, UnFinishedMetaBlocksIdentifier)
+		if err != nil {
+			return err
+		}
+	}
+
+	err = se.hardforkStorer.FinishedIdentifier(UnFinishedMetaBlocksIdentifier)
 	if err != nil {
 		return err
 	}
 
-	err = se.hardforkStorer.FinishedIdentifier(MetaBlockIdentifier)
+	return nil
+}
+
+func (se *stateExport) exportMetaBlock(metaBlock *block.MetaBlock, identifier string) error {
+	jsonData, err := json.Marshal(metaBlock)
 	if err != nil {
 		return err
 	}
 
-	log.Debug("Exported metaBlock", "rootHash", metaBlock.RootHash)
+	metaHash := se.hasher.Compute(string(jsonData))
+	versionKey := CreateVersionKey(metaBlock, metaHash)
+	err = se.hardforkStorer.Write(identifier, []byte(versionKey), jsonData)
+	if err != nil {
+		return err
+	}
+
+	log.Debug("Exported metaBlock",
+		"identifier", identifier,
+		"version key", versionKey,
+		"hash", metaHash,
+		"epoch", metaBlock.Epoch,
+		"round", metaBlock.Round,
+		"nonce", metaBlock.Nonce,
+		"start of epoch block", metaBlock.Nonce == 0 || metaBlock.IsStartOfEpochBlock(),
+		"rootHash", metaBlock.RootHash,
+	)
 
 	return nil
 }