Release Candidate v2023.3.0/v2023.3.1 - HIP30 Hard Fork

Makefile

@@ -56,6 +56,7 @@ trace-pointer:
 	bash ./scripts/go_executable_build.sh -t
 
 debug:
+	rm -rf .dht-127.0.0.1*
 	bash ./test/debug.sh
 
 debug-kill:
@@ -167,3 +168,15 @@ docker:
 
 travis_go_checker:
 	bash ./scripts/travis_go_checker.sh
+
+travis_rpc_checker:
+	bash ./scripts/travis_rpc_checker.sh
+
+travis_rosetta_checker:
+	bash ./scripts/travis_rosetta_checker.sh
+
+debug_external: clean
+	bash test/debug-external.sh
+
+build_localnet_validator:
+	bash test/build-localnet-validator.sh

accounts/abi/bind/auth.go

@@ -20,7 +20,6 @@ import (
 	"crypto/ecdsa"
 	"errors"
 	"io"
-	"io/ioutil"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -44,7 +43,7 @@ var ErrNotAuthorized = errors.New("not authorized to sign this account")
 // Deprecated: Use NewTransactorWithChainID instead.
 func NewTransactor(keyin io.Reader, passphrase string) (*TransactOpts, error) {
 	log.Warn("WARNING: NewTransactor has been deprecated in favour of NewTransactorWithChainID")
-	json, err := ioutil.ReadAll(keyin)
+	json, err := io.ReadAll(keyin)
 	if err != nil {
 		return nil, err
 	}
@@ -103,7 +102,7 @@ func NewKeyedTransactor(key *ecdsa.PrivateKey) *TransactOpts {
 // NewTransactorWithChainID is a utility method to easily create a transaction signer from
 // an encrypted json key stream and the associated passphrase.
 func NewTransactorWithChainID(keyin io.Reader, passphrase string, chainID *big.Int) (*TransactOpts, error) {
-	json, err := ioutil.ReadAll(keyin)
+	json, err := io.ReadAll(keyin)
 	if err != nil {
 		return nil, err
 	}

accounts/keystore/account_cache_test.go

@@ -18,7 +18,6 @@ package keystore
 
 import (
 	"fmt"
-	"io/ioutil"
 	"math/rand"
 	"os"
 	"path/filepath"
@@ -133,11 +132,11 @@ func TestUpdatedKeyfileContents(t *testing.T) {
 		return
 	}
 
-	// needed so that modTime of `file` is different to its current value after ioutil.WriteFile
+	// needed so that modTime of `file` is different to its current value after io.WriteFile
 	time.Sleep(1000 * time.Millisecond)
 
 	// Now replace file contents with crap
-	if err := ioutil.WriteFile(file, []byte("foo"), 0644); err != nil {
+	if err := os.WriteFile(file, []byte("foo"), 0644); err != nil {
 		t.Fatal(err)
 		return
 	}
@@ -150,9 +149,9 @@ func TestUpdatedKeyfileContents(t *testing.T) {
 
 // forceCopyFile is like cp.CopyFile, but doesn't complain if the destination exists.
 func forceCopyFile(dst, src string) error {
-	data, err := ioutil.ReadFile(src)
+	data, err := os.ReadFile(src)
 	if err != nil {
 		return err
 	}
-	return ioutil.WriteFile(dst, data, 0644)
+	return os.WriteFile(dst, data, 0644)
 }

accounts/keystore/file_cache.go

@@ -17,7 +17,7 @@
 package keystore
 
 import (
-	"io/ioutil"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
@@ -42,7 +42,7 @@ func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, er
 	t0 := time.Now()
 
 	// List all the failes from the keystore folder
-	files, err := ioutil.ReadDir(keyDir)
+	files, err := os.ReadDir(keyDir)
 	if err != nil {
 		return nil, nil, nil, err
 	}
@@ -63,15 +63,19 @@ func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, er
 			utils.Logger().Debug().Str("path", path).Msg("Ignoring file on account scan")
 			continue
 		}
-		// Gather the set of all and fresly modified files
+		// Gather the set of all and freshly modified files
 		all.Add(path)
 
-		modified := fi.ModTime()
-		if modified.After(fc.lastMod) {
-			mods.Add(path)
-		}
-		if modified.After(newLastMod) {
-			newLastMod = modified
+		if info, err := fi.Info(); err != nil {
+			continue
+		} else {
+			modified := info.ModTime()
+			if modified.After(fc.lastMod) {
+				mods.Add(path)
+			}
+			if modified.After(newLastMod) {
+				newLastMod = modified
+			}
 		}
 	}
 	t2 := time.Now()
@@ -94,14 +98,18 @@ func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, er
 }
 
 // nonKeyFile ignores editor backups, hidden files and folders/symlinks.
-func nonKeyFile(fi os.FileInfo) bool {
+func nonKeyFile(fi fs.DirEntry) bool {
 	// Skip editor backups and UNIX-style hidden files.
 	if strings.HasSuffix(fi.Name(), "~") || strings.HasPrefix(fi.Name(), ".") {
 		return true
 	}
 	// Skip misc special files, directories (yes, symlinks too).
-	if fi.IsDir() || fi.Mode()&os.ModeType != 0 {
+	if info, err := fi.Info(); err != nil {
 		return true
+	} else {
+		if fi.IsDir() || info.Mode()&os.ModeType != 0 {
+			return true
+		}
 	}
 	return false
 }

accounts/keystore/key.go

@@ -23,7 +23,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -195,7 +194,7 @@ func writeTemporaryKeyFile(file string, content []byte) (string, error) {
 	}
 	// Atomic write: create a temporary hidden file first
 	// then move it into place. TempFile assigns mode 0600.
-	f, err := ioutil.TempFile(filepath.Dir(file), "."+filepath.Base(file)+".tmp")
+	f, err := os.CreateTemp(filepath.Dir(file), "."+filepath.Base(file)+".tmp")
 	if err != nil {
 		return "", err
 	}

accounts/keystore/keystore_test.go

@@ -17,7 +17,6 @@
 package keystore
 
 import (
-	"io/ioutil"
 	"os"
 	"runtime"
 	"strings"
@@ -213,7 +212,7 @@ func TestSignRace(t *testing.T) {
 }
 
 func tmpKeyStore(t *testing.T, encrypted bool) (string, *KeyStore) {
-	d, err := ioutil.TempDir("", "eth-keystore-test")
+	d, err := os.MkdirTemp("", "eth-keystore-test")
 	if err != nil {
 		t.Fatal(err)
 	}

accounts/keystore/passphrase.go

@@ -34,7 +34,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 
@@ -82,7 +81,7 @@ type keyStorePassphrase struct {
 
 func (ks keyStorePassphrase) GetKey(addr common.Address, filename, auth string) (*Key, error) {
 	// Load the key from the keystore and decrypt its contents
-	keyjson, err := ioutil.ReadFile(filename)
+	keyjson, err := os.ReadFile(filename)
 	if err != nil {
 		return nil, err
 	}

accounts/keystore/passphrase_test.go

@@ -17,7 +17,7 @@
 package keystore
 
 import (
-	"io/ioutil"
+	"os"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -30,7 +30,7 @@ const (
 
 // Tests that a json key file can be decrypted and encrypted in multiple rounds.
 func TestKeyEncryptDecrypt(t *testing.T) {
-	keyjson, err := ioutil.ReadFile("testdata/very-light-scrypt.json")
+	keyjson, err := os.ReadFile("testdata/very-light-scrypt.json")
 	if err != nil {
 		t.Fatal(err)
 	}

accounts/keystore/plain_test.go

@@ -20,7 +20,6 @@ import (
 	"crypto/rand"
 	"encoding/hex"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -32,7 +31,7 @@ import (
 )
 
 func tmpKeyStoreIface(t *testing.T, encrypted bool) (dir string, ks keyStore) {
-	d, err := ioutil.TempDir("", "geth-keystore-test")
+	d, err := os.MkdirTemp("", "geth-keystore-test")
 	if err != nil {
 		t.Fatal(err)
 	}

api/service/legacysync/syncing.go

@@ -25,7 +25,6 @@ import (
 	"github.com/harmony-one/harmony/internal/chain"
 	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 	"github.com/harmony-one/harmony/internal/utils"
-	"github.com/harmony-one/harmony/node/worker"
 	"github.com/harmony-one/harmony/p2p"
 	libp2p_peer "github.com/libp2p/go-libp2p/core/peer"
 	"github.com/pkg/errors"
@@ -932,7 +931,7 @@ func (ss *StateSync) UpdateBlockAndStatus(block *types.Block, bc core.BlockChain
 }
 
 // generateNewState will construct most recent state from downloaded blocks
-func (ss *StateSync) generateNewState(bc core.BlockChain, worker *worker.Worker) error {
+func (ss *StateSync) generateNewState(bc core.BlockChain) error {
 	// update blocks created before node start sync
 	parentHash := bc.CurrentBlock().Hash()
 
@@ -995,7 +994,7 @@ func (ss *StateSync) generateNewState(bc core.BlockChain, worker *worker.Worker)
 }
 
 // ProcessStateSync processes state sync from the blocks received but not yet processed so far
-func (ss *StateSync) ProcessStateSync(startHash []byte, size uint32, bc core.BlockChain, worker *worker.Worker) error {
+func (ss *StateSync) ProcessStateSync(startHash []byte, size uint32, bc core.BlockChain) error {
 	// Gets consensus hashes.
 	if err := ss.getConsensusHashes(startHash, size); err != nil {
 		return errors.Wrap(err, "getConsensusHashes")
@@ -1005,7 +1004,7 @@ func (ss *StateSync) ProcessStateSync(startHash []byte, size uint32, bc core.Blo
 	if ss.stateSyncTaskQueue.Len() > 0 {
 		ss.downloadBlocks(bc)
 	}
-	return ss.generateNewState(bc, worker)
+	return ss.generateNewState(bc)
 }
 
 func (peerConfig *SyncPeerConfig) registerToBroadcast(peerHash []byte, ip, port string) error {
@@ -1076,7 +1075,7 @@ func (ss *StateSync) GetMaxPeerHeight() (uint64, error) {
 }
 
 // SyncLoop will keep syncing with peers until catches up
-func (ss *StateSync) SyncLoop(bc core.BlockChain, worker *worker.Worker, isBeacon bool, consensus *consensus.Consensus, loopMinTime time.Duration) {
+func (ss *StateSync) SyncLoop(bc core.BlockChain, isBeacon bool, consensus *consensus.Consensus, loopMinTime time.Duration) {
 	utils.Logger().Info().Msgf("legacy sync is executing ...")
 	if !isBeacon {
 		ss.RegisterNodeInfo()
@@ -1110,7 +1109,7 @@ func (ss *StateSync) SyncLoop(bc core.BlockChain, worker *worker.Worker, isBeaco
 		if size > SyncLoopBatchSize {
 			size = SyncLoopBatchSize
 		}
-		err := ss.ProcessStateSync(startHash[:], size, bc, worker)
+		err := ss.ProcessStateSync(startHash[:], size, bc)
 		if err != nil {
 			utils.Logger().Error().Err(err).
 				Msgf("[SYNC] ProcessStateSync failed (isBeacon: %t, ShardID: %d, otherHeight: %d, currentHeight: %d)",

api/service/stagedstreamsync/const.go

@@ -23,9 +23,6 @@ const (
 	// no more request will be assigned to workers to wait for InsertChain to finish.
 	SoftQueueCap int = 100
 
-	// DefaultConcurrency is the default settings for concurrency
-	DefaultConcurrency int = 4
-
 	// ShortRangeTimeout is the timeout for each short range sync, which allow short range sync
 	// to restart automatically when stuck in `getBlockHashes`
 	ShortRangeTimeout time.Duration = 1 * time.Minute
@@ -74,10 +71,10 @@ type (
 
 func (c *Config) fixValues() {
 	if c.Concurrency == 0 {
-		c.Concurrency = DefaultConcurrency
+		c.Concurrency = c.MinStreams
 	}
 	if c.Concurrency > c.MinStreams {
-		c.MinStreams = c.Concurrency
+		c.Concurrency = c.MinStreams
 	}
 	if c.MinStreams > c.InitStreams {
 		c.InitStreams = c.MinStreams

api/service/stagedstreamsync/downloader.go

@@ -153,6 +153,17 @@ func (d *Downloader) SubscribeDownloadFinished(ch chan struct{}) event.Subscript
 // waitForBootFinish waits for stream manager to finish the initial discovery and have
 // enough peers to start downloader
 func (d *Downloader) waitForBootFinish() {
+	bootCompleted, numStreams := d.waitForEnoughStreams(d.config.InitStreams)
+	if bootCompleted {
+		fmt.Printf("boot completed for shard %d ( %d streams are connected )\n",
+			d.bc.ShardID(), numStreams)
+	}
+}
+
+func (d *Downloader) waitForEnoughStreams(requiredStreams int) (bool, int) {
+	d.logger.Info().Int("requiredStreams", requiredStreams).
+		Msg("waiting for enough stream connections to continue syncing")
+
 	evtCh := make(chan streammanager.EvtStreamAdded, 1)
 	sub := d.syncProtocol.SubscribeAddStreamEvent(evtCh)
 	defer sub.Unsubscribe()
@@ -177,12 +188,11 @@ func (d *Downloader) waitForBootFinish() {
 			trigger()
 
 		case <-checkCh:
-			if d.syncProtocol.NumStreams() >= d.config.InitStreams {
-				fmt.Printf("boot completed for shard %d ( %d streams are connected )\n", d.bc.ShardID(), d.syncProtocol.NumStreams())
-				return
+			if d.syncProtocol.NumStreams() >= requiredStreams {
+				return true, d.syncProtocol.NumStreams()
 			}
 		case <-d.closeC:
-			return
+			return false, d.syncProtocol.NumStreams()
 		}
 	}
 }
@@ -212,6 +222,9 @@ func (d *Downloader) loop() {
 		case <-d.downloadC:
 			bnBeforeSync := d.bc.CurrentBlock().NumberU64()
 			estimatedHeight, addedBN, err := d.stagedSyncInstance.doSync(d.ctx, initSync)
+			if err == ErrNotEnoughStreams {
+				d.waitForEnoughStreams(d.config.MinStreams)
+			}
 			if err != nil {
 				//TODO: if there is a bad block which can't be resolved
 				if d.stagedSyncInstance.invalidBlock.Active {

api/service/stagedstreamsync/errors.go

@@ -14,7 +14,7 @@ var (
 	ErrUnexpectedNumberOfBlockHashes = WrapStagedSyncError("unexpected number of getBlocksByHashes result")
 	ErrUnexpectedBlockHashes         = WrapStagedSyncError("unexpected get block hashes result delivered")
 	ErrNilBlock                      = WrapStagedSyncError("nil block found")
-	ErrNotEnoughStreams              = WrapStagedSyncError("not enough streams")
+	ErrNotEnoughStreams              = WrapStagedSyncError("number of streams smaller than minimum required")
 	ErrParseCommitSigAndBitmapFail   = WrapStagedSyncError("parse commitSigAndBitmap failed")
 	ErrVerifyHeaderFail              = WrapStagedSyncError("verify header failed")
 	ErrInsertChainFail               = WrapStagedSyncError("insert to chain failed")

api/service/stagedstreamsync/short_range_helper.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/harmony-one/harmony/core/types"
+	"github.com/harmony-one/harmony/internal/utils"
 	syncProto "github.com/harmony-one/harmony/p2p/stream/protocols/sync"
 	sttypes "github.com/harmony-one/harmony/p2p/stream/types"
 	"github.com/pkg/errors"
@@ -132,6 +133,10 @@ func (sh *srHelper) getBlocksByHashes(ctx context.Context, hashes []common.Hash,
 
 func (sh *srHelper) checkPrerequisites() error {
 	if sh.syncProtocol.NumStreams() < sh.config.Concurrency {
+		utils.Logger().Info().
+			Int("available streams", sh.syncProtocol.NumStreams()).
+			Interface("concurrency", sh.config.Concurrency).
+			Msg("not enough streams to do concurrent processes")
 		return ErrNotEnoughStreams
 	}
 	return nil