fix(dot/state/epoch, lib/babe): enable block production through epochs without rely on finalization

dot/state/epoch.go

@@ -12,14 +12,17 @@ import (
 
 	"github.com/ChainSafe/chaindb"
 	"github.com/ChainSafe/gossamer/dot/types"
+	"github.com/ChainSafe/gossamer/lib/blocktree"
 	"github.com/ChainSafe/gossamer/lib/common"
 	"github.com/ChainSafe/gossamer/pkg/scale"
 )
 
 var (
-	ErrEpochNotInMemory = errors.New("epoch not found in memory map")
-	errHashNotInMemory  = errors.New("hash not found in memory map")
-	errHashNotPersisted = errors.New("hash with next epoch not found in database")
+	ErrEpochNotInMemory   = errors.New("epoch not found in memory map")
+	errHashNotInMemory    = errors.New("hash not found in memory map")
+	errEpochNotInDatabase = errors.New("epoch data not found in the database")
+	errConfigNotFound     = errors.New("config data not found")
+	errHashNotPersisted   = errors.New("hash with next epoch not found in database")
 )
 
 var (
@@ -251,14 +254,26 @@ func (s *EpochState) GetEpochData(epoch uint64, header *types.Header) (*types.Ep
 
 	if err != nil && !errors.Is(err, chaindb.ErrKeyNotFound) {
 		return nil, fmt.Errorf("failed to get epoch data from database: %w", err)
-	} else if header == nil {
-		// if no header is given then skip the lookup in-memory
-		return epochData, nil
 	}
 
-	epochData, err = s.getEpochDataFromMemory(epoch, header)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get epoch data from memory: %w", err)
+	//  lookup in-memory only if header is given
+	if header != nil && errors.Is(err, chaindb.ErrKeyNotFound) {
+		s.nextEpochDataLock.RLock()
+		defer s.nextEpochDataLock.RUnlock()
+
+		inMemoryEpochData, err := retrieveFromMemory(s.nextEpochData, s, epoch, header)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get epoch data from memory: %w", err)
+		}
+
+		epochData, err = inMemoryEpochData.ToEpochData()
+		if err != nil {
+			return nil, fmt.Errorf("cannot transform into epoch data: %w", err)
+		}
+	}
+
+	if epochData == nil {
+		return nil, errEpochNotInDatabase
 	}
 
 	return epochData, nil
@@ -280,32 +295,6 @@ func (s *EpochState) getEpochDataFromDatabase(epoch uint64) (*types.EpochData, e
 	return raw.ToEpochData()
 }
 
-// getEpochDataFromMemory retrieves the right epoch data that belongs to the header parameter
-func (s *EpochState) getEpochDataFromMemory(epoch uint64, header *types.Header) (*types.EpochData, error) {
-	s.nextEpochDataLock.RLock()
-	defer s.nextEpochDataLock.RUnlock()
-
-	atEpoch, has := s.nextEpochData[epoch]
-	if !has {
-		return nil, fmt.Errorf("%w: %d", ErrEpochNotInMemory, epoch)
-	}
-
-	headerHash := header.Hash()
-
-	for hash, value := range atEpoch {
-		isDescendant, err := s.blockState.IsDescendantOf(hash, headerHash)
-		if err != nil {
-			return nil, fmt.Errorf("cannot verify the ancestry: %w", err)
-		}
-
-		if isDescendant {
-			return value.ToEpochData()
-		}
-	}
-
-	return nil, fmt.Errorf("%w: %s", errHashNotInMemory, headerHash)
-}
-
 // GetLatestEpochData returns the EpochData for the current epoch
 func (s *EpochState) GetLatestEpochData() (*types.EpochData, error) {
 	curr, err := s.GetCurrentEpoch()
@@ -316,26 +305,6 @@ func (s *EpochState) GetLatestEpochData() (*types.EpochData, error) {
 	return s.GetEpochData(curr, nil)
 }
 
-// HasEpochData returns whether epoch data exists for a given epoch
-func (s *EpochState) HasEpochData(epoch uint64) (bool, error) {
-	has, err := s.db.Has(epochDataKey(epoch))
-	if err == nil && has {
-		return has, nil
-	}
-
-	// we can have `has == false` and `err == nil`
-	// so ensure the error is not nil in the condition below.
-	if err != nil && !errors.Is(chaindb.ErrKeyNotFound, err) {
-		return false, fmt.Errorf("cannot check database for epoch key %d: %w", epoch, err)
-	}
-
-	s.nextEpochDataLock.Lock()
-	defer s.nextEpochDataLock.Unlock()
-
-	_, has = s.nextEpochData[epoch]
-	return has, nil
-}
-
 // SetConfigData sets the BABE config data for a given epoch
 func (s *EpochState) SetConfigData(epoch uint64, info *types.ConfigData) error {
 	enc, err := scale.Marshal(*info)
@@ -357,28 +326,41 @@ func (s *EpochState) setLatestConfigData(epoch uint64) error {
 	return s.db.Put(latestConfigDataKey, buf)
 }
 
-// GetConfigData returns the config data for a given epoch persisted in database
-// otherwise tries to get the data from the in-memory map using the header.
-// If the header params is nil then it will search only in the database
-func (s *EpochState) GetConfigData(epoch uint64, header *types.Header) (*types.ConfigData, error) {
-	configData, err := s.getConfigDataFromDatabase(epoch)
-	if err == nil && configData != nil {
-		return configData, nil
+// GetConfigData returns the newest config data for a given epoch persisted in database
+// otherwise tries to get the data from the in-memory map using the header. If we don't
+// find any config data for the current epoch we lookup in the previous epochs, as the spec says:
+// - The supplied configuration data are intended to be used from the next epoch onwards.
+// If the header params is nil then it will search only in the database.
+func (s *EpochState) GetConfigData(epoch uint64, header *types.Header) (configData *types.ConfigData, err error) {
+	if header != nil {
+		s.nextConfigDataLock.RLock()
+		defer s.nextConfigDataLock.RUnlock()
 	}
 
-	if err != nil && !errors.Is(err, chaindb.ErrKeyNotFound) {
-		return nil, fmt.Errorf("failed to get config data from database: %w", err)
-	} else if header == nil {
-		// if no header is given then skip the lookup in-memory
-		return configData, nil
-	}
+	for tryEpoch := epoch; tryEpoch >= 0; tryEpoch-- {
+		configData, err = s.getConfigDataFromDatabase(tryEpoch)
+		if err == nil && configData != nil {
+			return configData, nil
+		}
 
-	configData, err = s.getConfigDataFromMemory(epoch, header)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get config data from memory: %w", err)
+		if err != nil && !errors.Is(err, chaindb.ErrKeyNotFound) {
+			return nil, fmt.Errorf("failed to get config data from database: %w", err)
+		}
+
+		//  lookup in-memory only if header is given
+		if header != nil && errors.Is(err, chaindb.ErrKeyNotFound) {
+			inMemoryConfigData, err := retrieveFromMemory(s.nextConfigData, s, epoch, header)
+			if errors.Is(err, ErrEpochNotInMemory) {
+				continue
+			} else if err != nil {
+				return nil, fmt.Errorf("failed to get config data from memory: %w", err)
+			}
+
+			return inMemoryConfigData.ToConfigData(), err
+		}
 	}
 
-	return configData, nil
+	return nil, errConfigNotFound
 }
 
 // getConfigDataFromDatabase returns the BABE config data for a given epoch persisted in database
@@ -397,26 +379,36 @@ func (s *EpochState) getConfigDataFromDatabase(epoch uint64) (*types.ConfigData,
 	return info, nil
 }
 
-// getConfigDataFromMemory retrieves the BABE config data for a given epoch that belongs to the header parameter
-func (s *EpochState) getConfigDataFromMemory(epoch uint64, header *types.Header) (*types.ConfigData, error) {
-	s.nextConfigDataLock.RLock()
-	defer s.nextConfigDataLock.RUnlock()
+func retrieveFromMemory[T types.NextEpochData | types.NextConfigData](nextEpochMap map[uint64]map[common.Hash]T,
+	es *EpochState, epoch uint64, header *types.Header) (*T, error) {
 
-	atEpoch, has := s.nextConfigData[epoch]
+	atEpoch, has := nextEpochMap[epoch]
 	if !has {
 		return nil, fmt.Errorf("%w: %d", ErrEpochNotInMemory, epoch)
 	}
 
 	headerHash := header.Hash()
-
 	for hash, value := range atEpoch {
-		isDescendant, err := s.blockState.IsDescendantOf(hash, headerHash)
+		isDescendant, err := es.blockState.IsDescendantOf(hash, headerHash)
+
+		// sometimes while moving to the next epoch is possible the header
+		// is not fully imported by the blocktree, in this case we will use
+		// its parent header which migth be already imported.
+		if errors.Is(err, blocktree.ErrEndNodeNotFound) {
+			parentHeader, err := es.blockState.GetHeader(header.ParentHash)
+			if err != nil {
+				return nil, fmt.Errorf("cannot get parent header: %w", err)
+			}
+
+			return retrieveFromMemory(nextEpochMap, es, epoch, parentHeader)
+		}
+
 		if err != nil {
 			return nil, fmt.Errorf("cannot verify the ancestry: %w", err)
 		}
 
 		if isDescendant {
-			return value.ToConfigData(), nil
+			return &value, nil
 		}
 	}
 
@@ -434,24 +426,6 @@ func (s *EpochState) GetLatestConfigData() (*types.ConfigData, error) {
 	return s.GetConfigData(epoch, nil)
 }
 
-// HasConfigData returns whether config data exists for a given epoch
-func (s *EpochState) HasConfigData(epoch uint64) (bool, error) {
-	has, err := s.db.Has(configDataKey(epoch))
-	if err == nil && has {
-		return has, nil
-	}
-
-	if err != nil && !errors.Is(chaindb.ErrKeyNotFound, err) {
-		return false, fmt.Errorf("cannot check database for epoch key %d: %w", epoch, err)
-	}
-
-	s.nextConfigDataLock.Lock()
-	defer s.nextConfigDataLock.Unlock()
-
-	_, has = s.nextConfigData[epoch]
-	return has, nil
-}
-
 // GetStartSlotForEpoch returns the first slot in the given epoch.
 // If 0 is passed as the epoch, it returns the start slot for the current epoch.
 func (s *EpochState) GetStartSlotForEpoch(epoch uint64) (uint64, error) {

dot/state/epoch_test.go

@@ -54,9 +54,6 @@ func TestEpochState_CurrentEpoch(t *testing.T) {
 
 func TestEpochState_EpochData(t *testing.T) {
 	s := newEpochStateFromGenesis(t)
-	has, err := s.HasEpochData(0)
-	require.NoError(t, err)
-	require.True(t, has)
 
 	keyring, err := keystore.NewSr25519Keyring()
 	require.NoError(t, err)

lib/babe/epoch.go

@@ -14,7 +14,7 @@ import (
 // initiateEpoch sets the epochData for the given epoch, runs the lottery for the slots in the epoch,
 // and stores updated EpochInfo in the database
 func (b *Service) initiateEpoch(epoch uint64) (*epochData, error) {
-	logger.Debugf("initiating epoch %d", epoch)
+	logger.Infof("initiating epoch %d", epoch)
 
 	// if epoch == 1, check that first slot is still set correctly
 	// ie. that the start slot of the network is the same as the slot number of block 1
@@ -24,15 +24,25 @@ func (b *Service) initiateEpoch(epoch uint64) (*epochData, error) {
 		}
 	}
 
-	epochData, startSlot, err := b.getEpochDataAndStartSlot(epoch)
+	bestBlockHeader, err := b.blockState.BestBlockHeader()
+	if err != nil {
+		return nil, fmt.Errorf("cannot get the best block header: %w", err)
+	}
+
+	epochData, err := b.getEpochData(epoch, bestBlockHeader)
 	if err != nil {
 		return nil, fmt.Errorf("cannot get epoch data and start slot: %w", err)
 	}
 
+	startSlot, err := b.epochState.GetStartSlotForEpoch(epoch)
+	if err != nil {
+		return nil, fmt.Errorf("cannot get start slot for epoch %d: %w", epoch, err)
+	}
+
 	// if we're at genesis, we need to determine when the first slot of the network will be
 	// by checking when we will be able to produce block 1.
 	// note that this assumes there will only be one producer of block 1
-	if b.blockState.BestBlockHash() == b.blockState.GenesisHash() {
+	if bestBlockHeader.Hash() == b.blockState.GenesisHash() {
 		startSlot, err = b.getFirstAuthoringSlot(epoch, epochData)
 		if err != nil {
 			return nil, fmt.Errorf("cannot get first authoring slot: %w", err)
@@ -75,78 +85,43 @@ func (b *Service) checkAndSetFirstSlot() error {
 	return nil
 }
 
-func (b *Service) getEpochDataAndStartSlot(epoch uint64) (*epochData, uint64, error) {
+func (b *Service) getEpochData(epoch uint64, bestBlock *types.Header) (*epochData, error) {
 	if epoch == 0 {
-		startSlot, err := b.epochState.GetStartSlotForEpoch(epoch)
-		if err != nil {
-			return nil, 0, fmt.Errorf("cannot get start slot for epoch %d: %w", epoch, err)
-		}
-
 		epochData, err := b.getLatestEpochData()
 		if err != nil {
-			return nil, 0, fmt.Errorf("cannot get latest epoch data: %w", err)
+			return nil, fmt.Errorf("cannot get latest epoch data: %w", err)
 		}
 
-		return epochData, startSlot, nil
-	}
-
-	has, err := b.epochState.HasEpochData(epoch)
-	if err != nil {
-		return nil, 0, fmt.Errorf("cannot check epoch state: %w", err)
-	}
-
-	if !has {
-		logger.Criticalf("%s number=%d", errNoEpochData, epoch)
-		return nil, 0, fmt.Errorf("%w: for epoch %d", errNoEpochData, epoch)
+		return epochData, nil
 	}
 
-	data, err := b.epochState.GetEpochData(epoch, nil)
+	currEpochData, err := b.epochState.GetEpochData(epoch, bestBlock)
 	if err != nil {
-		return nil, 0, fmt.Errorf("cannot get epoch data for epoch %d: %w", epoch, err)
+		return nil, fmt.Errorf("cannot get epoch data for epoch %d: %w", epoch, err)
 	}
 
-	idx, err := b.getAuthorityIndex(data.Authorities)
+	currentConfigData, err := b.epochState.GetConfigData(epoch, bestBlock)
 	if err != nil {
-		return nil, 0, fmt.Errorf("cannot get authority index: %w", err)
+		return nil, fmt.Errorf("cannot get config data for epoch %d: %w", epoch, err)
 	}
 
-	has, err = b.epochState.HasConfigData(epoch)
+	threshold, err := CalculateThreshold(currentConfigData.C1, currentConfigData.C2, len(currEpochData.Authorities))
 	if err != nil {
-		return nil, 0, fmt.Errorf("cannot check for config data for epoch %d: %w", epoch, err)
-	}
-
-	var cfgData *types.ConfigData
-	if has {
-		cfgData, err = b.epochState.GetConfigData(epoch, nil)
-		if err != nil {
-			return nil, 0, fmt.Errorf("cannot get config data for epoch %d: %w", epoch, err)
-		}
-	} else {
-		cfgData, err = b.epochState.GetLatestConfigData()
-		if err != nil {
-			return nil, 0, fmt.Errorf("cannot get latest config data from epoch state: %w", err)
-		}
+		return nil, fmt.Errorf("cannot calculate threshold: %w", err)
 	}
 
-	threshold, err := CalculateThreshold(cfgData.C1, cfgData.C2, len(data.Authorities))
+	idx, err := b.getAuthorityIndex(currEpochData.Authorities)
 	if err != nil {
-		return nil, 0, fmt.Errorf("cannot calculate threshold: %w", err)
+		return nil, fmt.Errorf("cannot get authority index: %w", err)
 	}
 
-	ed := &epochData{
-		randomness:     data.Randomness,
-		authorities:    data.Authorities,
+	return &epochData{
+		randomness:     currEpochData.Randomness,
+		authorities:    currEpochData.Authorities,
 		authorityIndex: idx,
 		threshold:      threshold,
-		allowedSlots:   types.AllowedSlots(cfgData.SecondarySlots),
-	}
-
-	startSlot, err := b.epochState.GetStartSlotForEpoch(epoch)
-	if err != nil {
-		return nil, 0, fmt.Errorf("cannot get start slot for epoch %d: %w", epoch, err)
-	}
-
-	return ed, startSlot, nil
+		allowedSlots:   types.AllowedSlots(currentConfigData.SecondarySlots),
+	}, nil
 }
 
 func (b *Service) getLatestEpochData() (resEpochData *epochData, error error) {