Merge branch 'devel' into tests11

.gitmodules

@@ -1,13 +1,6 @@
 [submodule "tests"]
 	path = tests/testdata
 	url = https://github.com/ethereum/tests
-[submodule "turbo/snapshotsync/snapshothashes/erigon-snapshots"]
-	path = turbo/snapshotsync/snapshothashes/erigon-snapshots
-	url = https://github.com/ledgerwatch/erigon-snapshot.git
-[submodule "cmd/downloader/trackers/trackerslist"]
-	path = cmd/downloader/trackers/trackerslist
-	url = https://github.com/ngosang/trackerslist.git
-
 [submodule "libmdbx"]
 	path = libmdbx
 	url = https://github.com/torquem-ch/libmdbx.git

Makefile

@@ -80,7 +80,7 @@ docker-compose: validate_docker_build_args setup_xdg_data_home
 dbg:
 	$(GO_DBG_BUILD) -o $(GOBIN)/ ./cmd/...
 
-%.cmd: git-submodules
+%.cmd:
 	@# Note: $* is replaced by the command name
 	@echo "Building $*"
 	@cd ./cmd/$* && $(GOBUILD) -o $(GOBIN)/$*

cmd/downloader/downloader/downloadercfg/logger.go

@@ -74,6 +74,9 @@ func (b adapterHandler) Handle(r lg.Record) {
 		if strings.Contains(str, "being sole dirtier of piece") { // suppress useless errors
 			break
 		}
+		if strings.Contains(str, "requested chunk too long") { // suppress useless errors
+			break
+		}
 
 		log.Warn(str)
 	case lg.Error:

cmd/downloader/trackers/embed.go

@@ -2,29 +2,18 @@ package trackers
 
 import (
 	"bufio"
-	_ "embed"
 	"strings"
-)
-
-//go:embed trackerslist/trackers_best.txt
-var best string
-var Best = split(best)
-
-//go:embed trackerslist/trackers_all_https.txt
-var https string
-var Https = split(https)
 
-//go:embed trackerslist/trackers_all_http.txt
-var http string
-var Http = split(http)
-
-//go:embed trackerslist/trackers_all_udp.txt
-var udp string
-var Udp = split(udp)
+	"github.com/ledgerwatch/trackerslist"
+)
 
-//go:embed trackerslist/trackers_all_ws.txt
-var ws string
-var Ws = split(ws)
+var (
+	Best  = split(trackerslist.Best)
+	Https = split(trackerslist.Https)
+	Http  = split(trackerslist.Http)
+	Udp   = split(trackerslist.Udp)
+	Ws    = split(trackerslist.Ws)
+)
 
 func split(txt string) (lines []string) {
 	sc := bufio.NewScanner(strings.NewReader(txt))

eth/stagedsync/stage_headers.go

@@ -27,7 +27,7 @@ import (
 	"github.com/ledgerwatch/erigon/turbo/engineapi"
 	"github.com/ledgerwatch/erigon/turbo/services"
 	"github.com/ledgerwatch/erigon/turbo/snapshotsync"
-	"github.com/ledgerwatch/erigon/turbo/snapshotsync/snapshothashes"
+	"github.com/ledgerwatch/erigon/turbo/snapshotsync/snapcfg"
 	"github.com/ledgerwatch/erigon/turbo/stages/bodydownload"
 	"github.com/ledgerwatch/erigon/turbo/stages/headerdownload"
 	"github.com/ledgerwatch/log/v3"
@@ -258,11 +258,11 @@ func startHandlingForkChoice(
 	cfg HeadersCfg,
 	headerInserter *headerdownload.HeaderInserter,
 ) (*privateapi.PayloadStatus, error) {
-	headerHash := forkChoice.HeadBlockHash
-	log.Debug(fmt.Sprintf("[%s] Handling fork choice", s.LogPrefix()), "headerHash", headerHash)
 	if cfg.memoryOverlay {
-		defer cfg.forkValidator.Clear(tx)
+		defer cfg.forkValidator.ClearWithUnwind(tx)
 	}
+	headerHash := forkChoice.HeadBlockHash
+	log.Debug(fmt.Sprintf("[%s] Handling fork choice", s.LogPrefix()), "headerHash", headerHash)
 
 	currentHeadHash := rawdb.ReadHeadHeaderHash(tx)
 	if currentHeadHash == headerHash { // no-op
@@ -577,10 +577,11 @@ func verifyAndSaveNewPoSHeader(
 	forkingHash, err := cfg.blockReader.CanonicalHash(ctx, tx, forkingPoint)
 
 	canExtendCanonical := forkingHash == currentHeadHash
-	canExtendFork := cfg.forkValidator.ExtendingForkHeadHash() == (common.Hash{}) || header.ParentHash == cfg.forkValidator.ExtendingForkHeadHash()
 
-	if cfg.memoryOverlay && (canExtendFork || header.ParentHash != currentHeadHash) {
-		status, latestValidHash, validationError, criticalError := cfg.forkValidator.ValidatePayload(tx, header, body, header.ParentHash == currentHeadHash /* extendCanonical */)
+	if cfg.memoryOverlay {
+		extendingHash := cfg.forkValidator.ExtendingForkHeadHash()
+		extendCanonical := (extendingHash == common.Hash{} && header.ParentHash == currentHeadHash) || extendingHash == header.ParentHash
+		status, latestValidHash, validationError, criticalError := cfg.forkValidator.ValidatePayload(tx, header, body, extendCanonical)
 		if criticalError != nil {
 			return nil, false, criticalError
 		}
@@ -664,6 +665,8 @@ func schedulePoSDownload(
 }
 
 func verifyAndSaveDownloadedPoSHeaders(tx kv.RwTx, cfg HeadersCfg, headerInserter *headerdownload.HeaderInserter) {
+	defer cfg.forkValidator.Clear()
+
 	var lastValidHash common.Hash
 	var badChainError error
 	var foundPow bool
@@ -1340,8 +1343,12 @@ func WaitForDownloader(ctx context.Context, cfg HeadersCfg, tx kv.RwTx) error {
 		}
 	}
 
+	if len(missingSnapshots) > 0 {
+		log.Warn("[Snapshots] downloading missing snapshots")
+	}
+
 	// send all hashes to the Downloader service
-	preverified := snapshothashes.KnownConfig(cfg.chainConfig.ChainName).Preverified
+	preverified := snapcfg.KnownCfg(cfg.chainConfig.ChainName).Preverified
 	i := 0
 	var downloadRequest []snapshotsync.DownloadRequest
 	// build all download requests
@@ -1361,7 +1368,6 @@ func WaitForDownloader(ctx context.Context, cfg HeadersCfg, tx kv.RwTx) error {
 	for _, r := range missingSnapshots {
 		downloadRequest = append(downloadRequest, snapshotsync.NewDownloadRequest(&r, "", ""))
 	}
-	req := snapshotsync.BuildProtoRequest(downloadRequest)
 
 	log.Info("[Snapshots] Fetching torrent files metadata")
 	for {
@@ -1370,7 +1376,7 @@ func WaitForDownloader(ctx context.Context, cfg HeadersCfg, tx kv.RwTx) error {
 			return ctx.Err()
 		default:
 		}
-		if _, err := cfg.snapshotDownloader.Download(ctx, req); err != nil {
+		if err := snapshotsync.RequestSnapshotDownload(ctx, downloadRequest, cfg.snapshotDownloader); err != nil {
 			log.Error("[Snapshots] call downloader", "err", err)
 			time.Sleep(10 * time.Second)
 			continue

eth/stagedsync/stage_senders.go

@@ -24,7 +24,7 @@ import (
 	"github.com/ledgerwatch/erigon/ethdb/prune"
 	"github.com/ledgerwatch/erigon/params"
 	"github.com/ledgerwatch/erigon/turbo/snapshotsync"
-	"github.com/ledgerwatch/erigon/turbo/snapshotsync/snapshothashes"
+	"github.com/ledgerwatch/erigon/turbo/snapshotsync/snapcfg"
 	"github.com/ledgerwatch/erigon/turbo/stages/headerdownload"
 	"github.com/ledgerwatch/log/v3"
 	"github.com/ledgerwatch/secp256k1"
@@ -42,7 +42,7 @@ type SendersCfg struct {
 	prune             prune.Mode
 	chainConfig       *params.ChainConfig
 	blockRetire       *snapshotsync.BlockRetire
-	snapshotHashesCfg *snapshothashes.Config
+	snapshotHashesCfg *snapcfg.Cfg
 	hd                *headerdownload.HeaderDownload
 }
 
@@ -62,7 +62,7 @@ func StageSendersCfg(db kv.RwDB, chainCfg *params.ChainConfig, badBlockHalt bool
 		chainConfig:       chainCfg,
 		prune:             prune,
 		blockRetire:       br,
-		snapshotHashesCfg: snapshothashes.KnownConfig(chainCfg.ChainName),
+		snapshotHashesCfg: snapcfg.KnownCfg(chainCfg.ChainName),
 		hd:                hd,
 	}
 }

go.mod

@@ -2,6 +2,14 @@ module github.com/ledgerwatch/erigon
 
 go 1.18
 
+require (
+	github.com/ledgerwatch/erigon-lib v0.0.0-20220720105945-114da7eca320
+	github.com/ledgerwatch/erigon-snapshot v1.0.0
+	github.com/ledgerwatch/log/v3 v3.4.1
+	github.com/ledgerwatch/secp256k1 v1.0.0
+	github.com/ledgerwatch/trackerslist v1.0.0
+)
+
 require (
 	github.com/RoaringBitmap/roaring v1.2.1
 	github.com/VictoriaMetrics/fastcache v1.10.0
@@ -36,9 +44,6 @@ require (
 	github.com/json-iterator/go v1.1.12
 	github.com/julienschmidt/httprouter v1.3.0
 	github.com/kevinburke/go-bindata v3.21.0+incompatible
-	github.com/ledgerwatch/erigon-lib v0.0.0-20220719040828-9ceeeac385ad
-	github.com/ledgerwatch/log/v3 v3.4.1
-	github.com/ledgerwatch/secp256k1 v1.0.0
 	github.com/nxadm/tail v1.4.9-0.20211216163028-4472660a31a6
 	github.com/pelletier/go-toml v1.9.5
 	github.com/pelletier/go-toml/v2 v2.0.2
@@ -50,7 +55,7 @@ require (
 	github.com/stretchr/testify v1.8.0
 	github.com/tendermint/go-amino v0.14.1
 	github.com/tendermint/tendermint v0.31.11
-	github.com/torquem-ch/mdbx-go v0.24.3-0.20220614090901-342411560dde
+	github.com/torquem-ch/mdbx-go v0.25.1-0.20220720103744-b96489e94ece
 	github.com/ugorji/go/codec v1.1.13
 	github.com/ugorji/go/codec/codecgen v1.1.13
 	github.com/urfave/cli v1.22.9

go.sum

@@ -390,12 +390,16 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0=
 github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
 github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
-github.com/ledgerwatch/erigon-lib v0.0.0-20220719040828-9ceeeac385ad h1:fdAdq41F6zH39l6FgsfezXZElEFzl80fXqnB7gKWCTE=
-github.com/ledgerwatch/erigon-lib v0.0.0-20220719040828-9ceeeac385ad/go.mod h1:KXCwHR5gW/dv9naTlrx4Du8Wzj6H3ndTBC+vw3hnyWU=
+github.com/ledgerwatch/erigon-lib v0.0.0-20220720105945-114da7eca320 h1:Wd2XPRsa/oVXz2j3a0554Ct8qAUS2IwZeTxdaCWzqbY=
+github.com/ledgerwatch/erigon-lib v0.0.0-20220720105945-114da7eca320/go.mod h1:mq8M03qcnaqXZ/yjNuWoyZQ5V8r5JbXw5JYmy4WNUZQ=
+github.com/ledgerwatch/erigon-snapshot v1.0.0 h1:bp/7xoPdM5lK7LFdqEMH008RZmqxMZV0RUVEQiWs7v4=
+github.com/ledgerwatch/erigon-snapshot v1.0.0/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
 github.com/ledgerwatch/log/v3 v3.4.1 h1:/xGwlVulXnsO9Uq+tzaExc8OWmXXHU0dnLalpbnY5Bc=
 github.com/ledgerwatch/log/v3 v3.4.1/go.mod h1:VXcz6Ssn6XEeU92dCMc39/g1F0OYAjw1Mt+dGP5DjXY=
 github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ=
 github.com/ledgerwatch/secp256k1 v1.0.0/go.mod h1:SPmqJFciiF/Q0mPt2jVs2dTr/1TZBTIA+kPMmKgBAak=
+github.com/ledgerwatch/trackerslist v1.0.0 h1:6gnQu93WCTL4jPcdmc8UEmw56Cb8IFQHLGnevfIeLwo=
+github.com/ledgerwatch/trackerslist v1.0.0/go.mod h1:pCC+eEw8izNcnBBiSwvIq8kKsxDLInAafSW275jqFrg=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
 github.com/lispad/go-generics-tools v1.1.0 h1:mbSgcxdFVmpoyso1X/MJHXbSbSL3dD+qhRryyxk+/XY=
@@ -619,8 +623,8 @@ github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDW
 github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/torquem-ch/mdbx-go v0.24.3-0.20220614090901-342411560dde h1:1nzKGldWC9T0ApRfV0jzH28DaBy1Yg5+rmjSiJ/G0dI=
-github.com/torquem-ch/mdbx-go v0.24.3-0.20220614090901-342411560dde/go.mod h1:T2fsoJDVppxfAPTLd1svUgH1kpPmeXdPESmroSHcL1E=
+github.com/torquem-ch/mdbx-go v0.25.1-0.20220720103744-b96489e94ece h1:jwLF5BKBWPb00kMfRmSHJl0Hwe52HonOVpNkBJZR+XI=
+github.com/torquem-ch/mdbx-go v0.25.1-0.20220720103744-b96489e94ece/go.mod h1:T2fsoJDVppxfAPTLd1svUgH1kpPmeXdPESmroSHcL1E=
 github.com/ugorji/go v1.1.13/go.mod h1:jxau1n+/wyTGLQoCkjok9r5zFa/FxT6eI5HiHKQszjc=
 github.com/ugorji/go/codec v1.1.13 h1:013LbFhocBoIqgHeIHKlV4JWYhqogATYWZhIcH0WHn4=
 github.com/ugorji/go/codec v1.1.13/go.mod h1:oNVt3Dq+FO91WNQ/9JnHKQP2QJxTzoN7wCBFCq1OeuU=

params/chainspecs/goerli.json

@@ -13,6 +13,7 @@
   "istanbulBlock": 1561651,
   "berlinBlock": 4460644,
   "londonBlock": 5062605,
+  "terminalTotalDifficulty": 10790000,
   "terminalBlockHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
   "clique": {
     "period": 15,

turbo/engineapi/fork_validator.go

@@ -84,6 +84,12 @@ func (fv *ForkValidator) ExtendingForkHeadHash() common.Hash {
 // NotifyCurrentHeight is to be called at the end of the stage cycle and repressent the last processed block.
 func (fv *ForkValidator) NotifyCurrentHeight(currentHeight uint64) {
 	fv.currentHeight = currentHeight
+	// If the head changed,e previous assumptions on head are incorrect now.
+	if fv.extendingFork != nil {
+		fv.extendingFork.Rollback()
+	}
+	fv.extendingFork = nil
+	fv.extendingForkHeadHash = common.Hash{}
 }
 
 // FlushExtendingFork flush the current extending fork if fcu chooses its head hash as the its forkchoice.
@@ -176,7 +182,16 @@ func (fv *ForkValidator) ValidatePayload(tx kv.RwTx, header *types.Header, body
 // Clear wipes out current extending fork data, this method is called after fcu is called,
 // because fcu decides what the head is and after the call is done all the non-chosed forks are
 // to be considered obsolete.
-func (fv *ForkValidator) Clear(tx kv.RwTx) {
+func (fv *ForkValidator) Clear() {
+	if fv.extendingFork != nil {
+		fv.extendingFork.Rollback()
+	}
+	fv.extendingForkHeadHash = common.Hash{}
+	fv.extendingFork = nil
+}
+
+// Clear wipes out current extending fork data and notify txpool.
+func (fv *ForkValidator) ClearWithUnwind(tx kv.RwTx) {
 	sb, ok := fv.sideForksBlock[fv.extendingForkHeadHash]
 	// If we did not flush the fork state, then we need to notify the txpool through unwind.
 	if fv.extendingFork != nil && fv.extendingForkHeadHash != (common.Hash{}) && ok {
@@ -187,8 +202,7 @@ func (fv *ForkValidator) Clear(tx kv.RwTx) {
 		}
 		fv.extendingFork.Rollback()
 	}
-	fv.extendingForkHeadHash = common.Hash{}
-	fv.extendingFork = nil
+	fv.Clear()
 }
 
 // validateAndStorePayload validate and store a payload fork chain if such chain results valid.
@@ -198,6 +212,11 @@ func (fv *ForkValidator) validateAndStorePayload(tx kv.RwTx, header *types.Heade
 	if validationError != nil {
 		latestValidHash = header.ParentHash
 		status = remote.EngineStatus_INVALID
+		if fv.extendingFork != nil {
+			fv.extendingFork.Rollback()
+			fv.extendingFork = nil
+		}
+		fv.extendingForkHeadHash = common.Hash{}
 		return
 	}
 	// If we do not have the body we can recover it from the batch.

turbo/snapshotsync/block_reader.go

@@ -516,7 +516,7 @@ func (back *BlockReaderWithSnapshots) headerFromSnapshot(blockHeight uint64, sn
 func (back *BlockReaderWithSnapshots) headerFromSnapshotByHash(hash common.Hash, sn *HeaderSegment, buf []byte) (*types.Header, error) {
 	defer func() {
 		if rec := recover(); rec != nil {
-			panic(fmt.Errorf("%+v, snapshot: %d-%d, trace: %s", rec, sn.From, sn.To, dbg.Stack()))
+			panic(fmt.Errorf("%+v, snapshot: %d-%d, trace: %s", rec, sn.ranges.from, sn.ranges.to, dbg.Stack()))
 		}
 	}() // avoid crash because Erigon's core does many things
 
@@ -564,7 +564,7 @@ func (back *BlockReaderWithSnapshots) bodyFromSnapshot(blockHeight uint64, sn *B
 func (back *BlockReaderWithSnapshots) bodyForStorageFromSnapshot(blockHeight uint64, sn *BodySegment, buf []byte) (*types.BodyForStorage, []byte, error) {
 	defer func() {
 		if rec := recover(); rec != nil {
-			panic(fmt.Errorf("%+v, snapshot: %d-%d, trace: %s", rec, sn.From, sn.To, dbg.Stack()))
+			panic(fmt.Errorf("%+v, snapshot: %d-%d, trace: %s", rec, sn.ranges.from, sn.ranges.to, dbg.Stack()))
 		}
 	}() // avoid crash because Erigon's core does many things
 
@@ -597,7 +597,7 @@ func (back *BlockReaderWithSnapshots) bodyForStorageFromSnapshot(blockHeight uin
 func (back *BlockReaderWithSnapshots) txsFromSnapshot(baseTxnID uint64, txsAmount uint32, txsSeg *TxnSegment, buf []byte) (txs []types.Transaction, senders []common.Address, err error) {
 	defer func() {
 		if rec := recover(); rec != nil {
-			panic(fmt.Errorf("%+v, snapshot: %d-%d, trace: %s", rec, txsSeg.From, txsSeg.To, dbg.Stack()))
+			panic(fmt.Errorf("%+v, snapshot: %d-%d, trace: %s", rec, txsSeg.ranges.from, txsSeg.ranges.to, dbg.Stack()))
 		}
 	}() // avoid crash because Erigon's core does many things