Merge pull request #5979 from onflow/bastian/improve-migration-determ…

…inism

Make more parts of the migration deterministic

cmd/util/ledger/migrations/account_based_migration.go

@@ -182,7 +182,7 @@ func MigrateGroupConcurrently(
 						continue
 					}
 
-					for m, migration := range migrations {
+					for migrationIndex, migration := range migrations {
 
 						select {
 						case <-ctx.Done():
@@ -194,7 +194,7 @@ func MigrateGroupConcurrently(
 						if err != nil {
 							log.Error().
 								Err(err).
-								Int("migration_index", m).
+								Int("migration_index", migrationIndex).
 								Type("migration", migration).
 								Hex("address", address[:]).
 								Msg("could not migrate account")

cmd/util/ledger/migrations/contract_checking_migration.go

@@ -2,6 +2,7 @@ package migrations
 
 import (
 	"fmt"
+	"sort"
 	"strings"
 
 	"github.com/onflow/cadence/runtime/common"
@@ -41,7 +42,13 @@ func NewContractCheckingMigration(
 
 		// Gather all contracts
 
-		contractsByLocation := make(map[common.Location][]byte, contractCountEstimate)
+		contractsForPrettyPrinting := make(map[common.Location][]byte, contractCountEstimate)
+
+		type contract struct {
+			location common.AddressLocation
+			code     []byte
+		}
+		contracts := make([]contract, 0, contractCountEstimate)
 
 		err = registersByAccount.ForEach(func(owner string, key string, value []byte) error {
 
@@ -58,17 +65,34 @@ func NewContractCheckingMigration(
 				Name:    contractName,
 			}
 
-			contractsByLocation[location] = code
+			contracts = append(
+				contracts,
+				contract{
+					location: location,
+					code:     code,
+				},
+			)
+
+			contractsForPrettyPrinting[location] = code
 
 			return nil
 		})
 		if err != nil {
 			return fmt.Errorf("failed to iterate over registers: %w", err)
 		}
 
+		sort.Slice(contracts, func(i, j int) bool {
+			a := contracts[i]
+			b := contracts[j]
+			return a.location.ID() < b.location.ID()
+		})
+
 		// Check all contracts
 
-		for location, code := range contractsByLocation {
+		for _, contract := range contracts {
+			location := contract.location
+			code := contract.code
+
 			log.Info().Msgf("checking contract %s ...", location)
 
 			// Check contract code
@@ -80,7 +104,7 @@ func NewContractCheckingMigration(
 				var builder strings.Builder
 				errorPrinter := pretty.NewErrorPrettyPrinter(&builder, false)
 
-				printErr := errorPrinter.PrettyPrintError(err, location, contractsByLocation)
+				printErr := errorPrinter.PrettyPrintError(err, location, contractsForPrettyPrinting)
 
 				var errorDetails string
 				if printErr == nil {
@@ -89,8 +113,6 @@ func NewContractCheckingMigration(
 					errorDetails = err.Error()
 				}
 
-				addressLocation := location.(common.AddressLocation)
-
 				if verboseErrorOutput {
 					log.Error().Msgf(
 						"error checking contract %s: %s",
@@ -100,8 +122,8 @@ func NewContractCheckingMigration(
 				}
 
 				reporter.Write(contractCheckingFailure{
-					AccountAddressHex: addressLocation.Address.HexWithPrefix(),
-					ContractName:      addressLocation.Name,
+					AccountAddressHex: location.Address.HexWithPrefix(),
+					ContractName:      location.Name,
 					Error:             errorDetails,
 				})
 

cmd/util/ledger/migrations/filter_unreferenced_slabs_migration.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"path"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -128,7 +129,20 @@ func (m *FilterUnreferencedSlabsMigration) MigrateAccount(
 		Str("account", address.HexWithPrefix()).
 		Msgf("filtering %d unreferenced slabs", len(unreferencedSlabIDs))
 
+	var storageIDs []atree.StorageID
 	for storageID := range unreferencedSlabIDs {
+		storageIDs = append(storageIDs, storageID)
+	}
+	sort.Slice(
+		storageIDs,
+		func(i, j int) bool {
+			a := storageIDs[i]
+			b := storageIDs[j]
+			return a.Compare(b) < 0
+		},
+	)
+
+	for _, storageID := range storageIDs {
 		owner, key := registerFromStorageID(storageID)
 
 		value, err := accountRegisters.Get(owner, key)

cmd/util/ledger/migrations/fix_broken_data_migration.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"path"
+	"sort"
 	"sync"
 	"time"
 
@@ -215,7 +216,10 @@ func getAtreePayloadsByID(
 ) {
 	outputPayloads := make([]*ledger.Payload, 0, len(ids))
 
-	err := registers.ForEach(func(owner string, key string, value []byte) error {
+	owner := registers.Owner()
+
+	keys := make([]string, 0, len(ids))
+	err := registers.ForEachKey(func(key string) error {
 
 		if !flow.IsSlabIndexKey(key) {
 			return nil
@@ -231,17 +235,28 @@ func getAtreePayloadsByID(
 			return nil
 		}
 
+		keys = append(keys, key)
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	sort.Strings(keys)
+
+	for _, key := range keys {
+		value, err := registers.Get(owner, key)
+		if err != nil {
+			return nil, err
+		}
+
 		ledgerKey := convert.RegisterIDToLedgerKey(flow.RegisterID{
 			Owner: owner,
 			Key:   key,
 		})
 		payload := ledger.NewPayload(ledgerKey, value)
 		outputPayloads = append(outputPayloads, payload)
-
-		return nil
-	})
-	if err != nil {
-		return nil, err
 	}
 
 	return outputPayloads, nil

cmd/util/ledger/migrations/staged_contracts_migration.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"sort"
 	"strings"
 	"sync"
 
@@ -425,7 +426,14 @@ func (m *StagedContractsMigration) MigrateAccount(
 		return nil
 	}
 
-	for name, contract := range contractUpdates {
+	var contractNames []string
+	for name := range contractUpdates {
+		contractNames = append(contractNames, name)
+	}
+	sort.Strings(contractNames)
+
+	for _, name := range contractNames {
+		contract := contractUpdates[name]
 
 		owner := string(address[:])
 		key := flow.ContractKey(name)

cmd/util/ledger/util/registers/registers.go

@@ -277,6 +277,16 @@ func (a *AccountRegisters) Merge(other *AccountRegisters) error {
 	return nil
 }
 
+func (a *AccountRegisters) ForEachKey(f func(key string) error) error {
+	for key := range a.registers {
+		err := f(key)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func NewAccountRegistersFromPayloads(owner string, payloads []*ledger.Payload) (*AccountRegisters, error) {
 	accountRegisters := NewAccountRegisters(owner)