More updates to downloader, new p2psentry protocol (ledgerwatch#1559)

* Initial commit * Add sentry gRPC interface * p2psentry directory * Update README.md * Update README.md * Update README.md * Add go package * Correct syntax * add external downloader interface (#2) * Add txpool (#3) * Add private API (#4) * Invert control.proto, add PeerMinBlock, Separare incoming Tx message into a separate stream (#5) Co-authored-by: Alexey Sharp <alexeysharp@Alexeys-iMac.local> * Separate upload messages into its own stream (ledgerwatch#6) Co-authored-by: Alexey Sharp <alexeysharp@Alexeys-iMac.local> * Only send changed accounts to listeners (ledgerwatch#7) * Txpool interface doc (ledgerwatch#9) * More additions * More additions * Fix locking * Intermediate * Fix separation of phases * Intermediate * Fix test * More transformations * New simplified way of downloading headers * Fix hard-coded header sync * Fixed syncing near the tip of the chain * Add architecture diagram source and picture (ledgerwatch#10) * More fixes * rename tip to link * Use preverified hashes instead of preverified headers * Fix preverified hashes generation * more parametrisation * Continue parametrisation * Fix grpc data limit, interruption of headers stage * Add ropsten preverified hashes * Typed hashes (ledgerwatch#11) * Typed hashes * Fix PeerId * 64-bit tx nonce * Disable penalties * Add goerli settings, bootstrap nodes * Try to fix goerly sync * Remove interfaces * Add proper golang packages, max_block into p2p sentry Status * Prepare for proto overhaul * Squashed 'interfaces/' content from commit ce36053 git-subtree-dir: interfaces git-subtree-split: ce36053 * Change EtherReply to address * Adaptations to new types * Switch to new types * Fixes * Fix formatting * Fix lint * Lint fixes, reverse order in types * Fix lint * Fix lint * Fix lint * Fix test * Not supporting eth/66 yet * Fix shutdown * Fix lint * Fix lint * Fix lint * return stopped check Co-authored-by: Artem Vorotnikov <artem@vorotnikov.me> Co-authored-by: b00ris <b00ris@mail.ru> Co-authored-by: Alexey Sharp <alexeysharp@Alexeys-iMac.local> Co-authored-by: lightclient <14004106+lightclient@users.noreply.github.com> Co-authored-by: canepat <16927169+canepat@users.noreply.github.com>
contractshark · Apr 5, 2021 · b019c2a · b019c2a
1 parent 2882e03
commit b019c2a
Show file tree

Hide file tree

Showing 61 changed files with 116,875 additions and 3,059 deletions.
diff --git a/Makefile b/Makefile
@@ -177,10 +177,12 @@ grpc:
 
 	$(GOBUILD) -o $(GOBIN)/protoc-gen-go google.golang.org/protobuf/cmd/protoc-gen-go # generates proto messages
 	$(GOBUILD) -o $(GOBIN)/protoc-gen-go-grpc google.golang.org/grpc/cmd/protoc-gen-go-grpc # generates grpc services
-	PATH=$(GOBIN):$(PATH) go generate ./ethdb
-	PATH=$(GOBIN):$(PATH) go generate ./cmd/headers
-	PATH=$(GOBIN):$(PATH) go generate ./turbo/shards
-	PATH=$(GOBIN):$(PATH) go generate ./turbo/snapshotsync
+	PATH=$(GOBIN):$(PATH) protoc --proto_path=interfaces --go_out=gointerfaces --go-grpc_out=gointerfaces -I=build/include/google \
+		--go_opt=Mtypes/types.proto=github.com/ledgerwatch/turbo-geth/gointerfaces/types \
+		types/types.proto \
+		p2psentry/sentry.proto \
+		remote/kv.proto remote/db.proto remote/ethbackend.proto \
+		snapshot_downloader/external_downloader.proto
 
 prometheus:
 	docker-compose up prometheus grafana

diff --git a/cmd/hack/hack.go b/cmd/hack/hack.go
@@ -1641,6 +1641,43 @@ func mint(chaindata string, block uint64) error {
 	return nil
 }
 
+func extractHashes(chaindata string, blockStep uint64, blockTotal uint64, name string) error {
+	db := ethdb.MustOpen(chaindata)
+	defer db.Close()
+
+	f, err := os.Create(fmt.Sprintf("preverified_hashes_%s.go", name))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	w := bufio.NewWriter(f)
+	defer w.Flush()
+
+	fmt.Fprintf(w, "package headerdownload\n\n")
+	fmt.Fprintf(w, "var %sPreverifiedHashes = []string{\n", name)
+
+	b := uint64(0)
+	for b <= blockTotal {
+		hash, err := rawdb.ReadCanonicalHash(db, b)
+		if err != nil {
+			return err
+		}
+
+		if hash == (common.Hash{}) {
+			break
+		}
+
+		fmt.Fprintf(w, "	\"%x\",\n", hash)
+		b += blockStep
+	}
+	b -= blockStep
+	fmt.Fprintf(w, "}\n\n")
+	fmt.Fprintf(w, "const %sPreverifiedHeight uint64 = %d\n", name, b)
+	fmt.Printf("Last block is %d\n", b)
+	return nil
+}
+
 func extractHeaders(chaindata string, blockStep uint64, blockTotal uint64, name string) error {
 	db := ethdb.MustOpen(chaindata)
 	defer db.Close()
@@ -1984,6 +2021,11 @@ func main() {
 			fmt.Printf("Error: %v\n", err)
 		}
 	}
+	if *action == "extractHashes" {
+		if err := extractHashes(*chaindata, uint64(*block), uint64(*blockTotal), *name); err != nil {
+			fmt.Printf("Error: %v\n", err)
+		}
+	}
 	if *action == "defrag" {
 		if err := db.Defrag(); err != nil {
 			fmt.Printf("Error: %v\n", err)

diff --git a/cmd/headers/check/check.go b/cmd/headers/check/check.go
diff --git a/cmd/headers/commands/check.go b/cmd/headers/commands/check.go
diff --git a/cmd/headers/commands/download.go b/cmd/headers/commands/download.go
@@ -6,19 +6,18 @@ import (
 )
 
 var (
-	bufferSizeStr string // Size of buffer
-	combined      bool   // Whether downloader also includes sentry
-	timeout       int    // Timeout for delivery requests
-	window        int    // Size of sliding window for downloading block bodies
+	combined bool   // Whether downloader also includes sentry
+	timeout  int    // Timeout for delivery requests
+	window   int    // Size of sliding window for downloading block bodies
+	chain    string // Name of the network to connect to
 )
 
 func init() {
-	downloadCmd.Flags().StringVar(&filesDir, "filesdir", "", "path to directory where files will be stored")
-	downloadCmd.Flags().StringVar(&bufferSizeStr, "bufferSize", "512M", "size o the buffer")
 	downloadCmd.Flags().StringVar(&sentryAddr, "sentryAddr", "localhost:9091", "sentry address <host>:<port>")
 	downloadCmd.Flags().BoolVar(&combined, "combined", false, "run downloader and sentry in the same process")
 	downloadCmd.Flags().IntVar(&timeout, "timeout", 30, "timeout for devp2p delivery requests, in seconds")
 	downloadCmd.Flags().IntVar(&window, "window", 65536, "size of sliding window for downloading block bodies, block")
+	downloadCmd.Flags().StringVar(&chain, "chain", "mainnet", "Name of the network (mainnet, testnets) to connect to")
 
 	// Options below are only used in the combined mode
 	downloadCmd.Flags().StringVar(&natSetting, "nat", "any", "NAT port mapping mechanism (any|none|upnp|pmp|extip:<IP>)")
@@ -39,8 +38,8 @@ var downloadCmd = &cobra.Command{
 		db := openDatabase(chaindata)
 		defer db.Close()
 		if combined {
-			return download.Combined(natSetting, port, staticPeers, discovery, netRestrict, filesDir, bufferSizeStr, db, timeout, window)
+			return download.Combined(natSetting, port, staticPeers, discovery, netRestrict, db, timeout, window, chain)
 		}
-		return download.Download(filesDir, bufferSizeStr, sentryAddr, coreAddr, db, timeout, window)
+		return download.Download(sentryAddr, coreAddr, db, timeout, window, chain)
 	},
 }
diff --git a/cmd/headers/commands/root.go b/cmd/headers/commands/root.go
@@ -16,7 +16,6 @@ import (
 )
 
 var (
-	filesDir      string // Directory when the files should be stored
 	sentryAddr    string // Address of the sentry <host>:<port>
 	coreAddr      string // Address of the core <host>:<port>
 	chaindata     string // Path to chaindata