Merge pull request #571 from gohornet/develop

Release 0.4.2

.github/workflows/build_docker.yml

@@ -21,4 +21,4 @@ jobs:
         run: docker build . --file docker/Dockerfile --tag hornet:latest
 
       - name: Test HORNET Docker image
-        run: docker run --rm --name hornet hornet:latest --help 2>/dev/null | grep -q "help requested"
+        run: docker run --rm --name hornet hornet:latest --version 2>/dev/null | grep -q "HORNET"

CHANGELOG.md

@@ -2,6 +2,78 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.4.2] - 22.06.2020
+
+### Added
+
+    - Snapshot download fallback sources (#568)
+
+### Changed
+
+    - Using --version instead of --help for checking if the docker image works
+    - Update autopeering entry nodes
+
+### Fixed
+
+    - Hiding of all non essential config flags
+    - Ignoring all entry nodes if one is not found
+    - Explicit --help/-h flag to print the help instead of using the built-in missing help flag error handling of pflag
+    - Missing trytes convertion to ledger panic logs
+
+### Config file changes
+
+`config.json`
+
+```diff
+- "downloadURL": "https://ls.manapotion.io/export.bin"
++ "downloadURLs": [
++   "https://ls.manapotion.io/export.bin",
++   "https://x-vps.com/export.bin",
++   "https://dbfiles.iota.org/mainnet/hornet/latest-export.bin"
++ ]
+
+  "entryNodes": [
+-   "46CstniGgfWMdAySiWuS7bVfugwuHZCUQKVaC4Y34EYJ@enter.hornet.zone:14626",
++   "FvfwJuCMoWJvcJLSYww7whPxouZ9WFJ55uyxTxKxJ1ez@enter.hornet.zone:14626",
+-   "2GHfjJhTqRaKCGBJJvS5RWty61XhjX7FtbVDhg7s8J1x@entrynode.tanglebay.org:14626",
+-   "iotaMk9Rg8wWo1DDeG7fwV9iJ41hvkwFX8w6MyTQgDu@enter.thetangle.org:14627"
++   "iotaMk9Rg8wWo1DDeG7fwV9iJ41hvkwFX8w6MyTQgDu@enter.thetangle.org:14627",
++   "12w9FrzMdDQ42aBgFrv1siHuJMhuZ4SMVHRFSS7Zb72W@entrynode.iotatoken.nl:14626",
++   "DboTc1v61Xdyvggj8VRszy92ScUTLgfwZaHvXsU8zr7e@entrynode.tanglebay.org:14626"
+  ],
+```
+
+`config_comnet.json`
+
+```diff
+- "downloadURL": "https://ls.tanglebay.org/comnet/export.bin"
++ "downloadURLs": [
++   "https://ls.manapotion.io/comnet/export.bin"
++ ]
+
+  "entryNodes": [
+-   "7Y1GSTTwJLMPCffNJhWggZPtwVce5hsgAVcHanNa6HXh@entrynode.comnet.tanglebay.org:14636",
+-   "FPE6kHwZhvw8g163faJwTaPzYePbYtaXhwpWxFKuJfEY@enter.comnet.hornet.zone:14627"
++   "GLZAWBGqvm6ZRT7jGMFAKyUJNPdvx4i5A1GPRZbGS6C9@enter.comnet.hornet.zone:14627",
++   "J1Hn5r9pS5FkLeYqXWstC2Zyjxj73grEWvjuene3qjM9@entrynode.comnet.tanglebay.org:14636"
+  ],
+```
+
+`config_devnet.json`
+
+```diff
+- "downloadURL": "https://dbfiles.iota.org/devnet/hornet/latest-export.bin"
++ "downloadURLs": [
++   "https://dbfiles.iota.org/devnet/hornet/latest-export.bin"
++ ]
+
+  "entryNodes": [
+-   "iotaDvNxMP5EPQPbHNzMTZK5ipd4BGZfjZBomenmyk3@enter.devnet.thetangle.org:14637"
++   "iotaDvNxMP5EPQPbHNzMTZK5ipd4BGZfjZBomenmyk3@enter.devnet.thetangle.org:14637",
++   "BqXajrWBFGYcJduK7kxiSMW3hv9fXRLzt9jK7JZZPAzp@entrynode.devnet.tanglebay.org:14646"
+  ],
+```
+
 ## [0.4.1] - 30.06.2020
 
 ### Added

config.json

@@ -51,7 +51,11 @@
       "intervalSynced": 50,
       "intervalUnsynced": 1000,
       "path": "snapshots/mainnet/export.bin",
-      "downloadURL": "https://ls.manapotion.io/export.bin"
+      "downloadURLs": [
+        "https://ls.manapotion.io/export.bin",
+        "https://x-vps.com/export.bin",
+        "https://dbfiles.iota.org/mainnet/hornet/latest-export.bin"
+      ]
     },
     "global": {
       "path": "snapshotMainnet.txt",
@@ -90,10 +94,11 @@
       "bindAddress": "0.0.0.0:14626",
       "runAsEntryNode": false,
       "entryNodes": [
-        "46CstniGgfWMdAySiWuS7bVfugwuHZCUQKVaC4Y34EYJ@enter.hornet.zone:14626",
+        "FvfwJuCMoWJvcJLSYww7whPxouZ9WFJ55uyxTxKxJ1ez@enter.hornet.zone:14626",
         "EkSLZ4uvSTED1x6KaGzqxoGxjbytt2rPVfbJk1LRLCGL@enter.manapotion.io:18626",
-        "2GHfjJhTqRaKCGBJJvS5RWty61XhjX7FtbVDhg7s8J1x@entrynode.tanglebay.org:14626",
-        "iotaMk9Rg8wWo1DDeG7fwV9iJ41hvkwFX8w6MyTQgDu@enter.thetangle.org:14627"
+        "iotaMk9Rg8wWo1DDeG7fwV9iJ41hvkwFX8w6MyTQgDu@enter.thetangle.org:14627",
+        "12w9FrzMdDQ42aBgFrv1siHuJMhuZ4SMVHRFSS7Zb72W@entrynode.iotatoken.nl:14626",
+        "DboTc1v61Xdyvggj8VRszy92ScUTLgfwZaHvXsU8zr7e@entrynode.tanglebay.org:14626"
       ],
       "seed": ""
     }

config_comnet.json

@@ -51,7 +51,9 @@
       "intervalSynced": 200,
       "intervalUnsynced": 1000,
       "path": "snapshots/comnet/export.bin",
-      "downloadURL": "https://ls.tanglebay.org/comnet/export.bin"
+      "downloadURLs": [
+        "https://ls.manapotion.io/comnet/export.bin"
+      ]
     },
     "global": {
       "path": "snapshot.csv",
@@ -87,8 +89,8 @@
       "runAsEntryNode": false,
       "entryNodes": [
         "iotaCrvEWGfaeA1HutcULjD4uZnPhEnD5xNGfGs8vhe@enter.comnet.thetangle.org:14647",
-        "7Y1GSTTwJLMPCffNJhWggZPtwVce5hsgAVcHanNa6HXh@entrynode.comnet.tanglebay.org:14636",
-        "FPE6kHwZhvw8g163faJwTaPzYePbYtaXhwpWxFKuJfEY@enter.comnet.hornet.zone:14627"
+        "GLZAWBGqvm6ZRT7jGMFAKyUJNPdvx4i5A1GPRZbGS6C9@enter.comnet.hornet.zone:14627",
+        "J1Hn5r9pS5FkLeYqXWstC2Zyjxj73grEWvjuene3qjM9@entrynode.comnet.tanglebay.org:14636"
       ],
       "seed": ""
     }

config_devnet.json

@@ -51,7 +51,9 @@
             "intervalSynced": 50,
             "intervalUnsynced": 1000,
             "path": "snapshots/devnet/export.bin",
-            "downloadURL": "https://dbfiles.iota.org/devnet/hornet/latest-export.bin"
+            "downloadURLs": [
+                "https://dbfiles.iota.org/devnet/hornet/latest-export.bin"
+            ]
         },
         "global": {
             "path": "snapshotDevnet.txt",
@@ -90,7 +92,8 @@
             "bindAddress": "0.0.0.0:14626",
             "runAsEntryNode": false,
             "entryNodes": [
-                "iotaDvNxMP5EPQPbHNzMTZK5ipd4BGZfjZBomenmyk3@enter.devnet.thetangle.org:14637"
+                "iotaDvNxMP5EPQPbHNzMTZK5ipd4BGZfjZBomenmyk3@enter.devnet.thetangle.org:14637",
+                "BqXajrWBFGYcJduK7kxiSMW3hv9fXRLzt9jK7JZZPAzp@entrynode.devnet.tanglebay.org:14646"
             ],
             "seed": ""
         }

main.go

@@ -29,6 +29,7 @@ import (
 )
 
 func main() {
+	cli.HideConfigFlags()
 	cli.PrintVersion()
 	cli.ParseConfig()
 	toolset.HandleTools()

pkg/config/config.go

@@ -47,6 +47,14 @@ var (
 	}
 )
 
+// HideConfigFlags hides all non essential flags from the help/usage text.
+func HideConfigFlags() {
+	flag.VisitAll(func(f *flag.Flag) {
+		_, notHidden := nonHiddenFlags[f.Name]
+		f.Hidden = !notHidden
+	})
+}
+
 // FetchConfig fetches config values from a dir defined via CLI flag --config-dir (or the current working dir if not set).
 //
 // It automatically reads in a single config file starting with "config" (can be changed via the --config CLI flag)
@@ -64,12 +72,6 @@ func FetchConfig() error {
 	PeeringConfig.AutomaticEnv()
 	ProfilesConfig.AutomaticEnv()
 
-	// hide all but the most essential flags
-	flag.VisitAll(func(f *flag.Flag) {
-		_, notHidden := nonHiddenFlags[f.Name]
-		f.Hidden = !notHidden
-	})
-
 	err := parameter.LoadConfigFile(NodeConfig, *configDirPath, *configName, true, !hasFlag(defaultConfigName))
 	if err != nil {
 		return err

pkg/config/snapshot_config.go

@@ -16,7 +16,7 @@ const (
 	// path to the local snapshot file
 	CfgLocalSnapshotsPath = "snapshots.local.path"
 	// URL to load the local snapshot file from
-	CfgLocalSnapshotsDownloadURL = "snapshots.local.downloadURL"
+	CfgLocalSnapshotsDownloadURLs = "snapshots.local.downloadURLs"
 	// path to the global snapshot file containing the ledger state
 	CfgGlobalSnapshotPath = "snapshots.global.path"
 	// paths to the spent addresses files
@@ -37,7 +37,7 @@ func init() {
 	flag.Int(CfgLocalSnapshotsIntervalSynced, 50, "interval, in milestone transactions, at which snapshot files are created if the ledger is fully synchronized")
 	flag.Int(CfgLocalSnapshotsIntervalUnsynced, 1000, "interval, in milestone transactions, at which snapshot files are created if the ledger is not fully synchronized")
 	flag.String(CfgLocalSnapshotsPath, "snapshots/mainnet/export.bin", "path to the local snapshot file")
-	flag.String(CfgLocalSnapshotsDownloadURL, "", "URL to load the local snapshot file from")
+	flag.StringSlice(CfgLocalSnapshotsDownloadURLs, []string{}, "URLs to load the local snapshot file from. Provide multiple URLs as fall back sources")
 	flag.String(CfgGlobalSnapshotPath, "snapshotMainnet.txt", "path to the global snapshot file containing the ledger state")
 	flag.StringSlice(CfgGlobalSnapshotSpentAddressesPaths, []string{
 		"previousEpochsSpentAddresses1.txt",

pkg/model/tangle/ledger_db.go

@@ -224,7 +224,7 @@ func GetLedgerStateForMilestoneWithoutLocking(targetIndex milestone.Index, abort
 			newBalance := int64(balances[address]) - change
 
 			if newBalance < 0 {
-				return nil, 0, fmt.Errorf("Ledger diff for milestone %d creates negative balance for address %s: current %d, diff %d", milestoneIndex, address, balances[address], change)
+				return nil, 0, fmt.Errorf("Ledger diff for milestone %d creates negative balance for address %s: current %d, diff %d", milestoneIndex, hornet.Hash(address).Trytes(), balances[address], change)
 			} else if newBalance == 0 {
 				delete(balances, address)
 			} else {
@@ -262,7 +262,7 @@ func ApplyLedgerDiffWithoutLocking(diff map[string]int64, index milestone.Index)
 		newBalance := int64(balance) + change
 
 		if newBalance < 0 {
-			panic(fmt.Sprintf("Ledger diff for milestone %d creates negative balance for address %s: current %d, diff %d", index, address, balance, change))
+			panic(fmt.Sprintf("Ledger diff for milestone %d creates negative balance for address %s: current %d, diff %d", index, hornet.Hash(address).Trytes(), balance, change))
 		} else if newBalance > 0 {
 			// Save balance
 			balanceBatch.Set(databaseKeyForAddress(hornet.Hash(address)), bytesFromBalance(uint64(newBalance)))

plugins/autopeering/autopeering.go

@@ -42,9 +42,8 @@ var (
 func configureAP(local *Local) {
 	entryNodes, err := parseEntryNodes()
 	if err != nil {
-		log.Errorf("Invalid entry nodes; ignoring: %v", err)
+		log.Warn(err)
 	}
-	log.Debugf("Entry node peers: %v", entryNodes)
 
 	gossipServiceKeyHash := fnv.New32a()
 	gossipServiceKeyHash.Write([]byte(services.GossipServiceKey()))
@@ -122,42 +121,56 @@ func start(local *Local, shutdownSignal <-chan struct{}) {
 	log.Info("Stopping Autopeering ...")
 }
 
-func parseEntryNodes() (result []*peer.Peer, err error) {
-	for _, entryNodeDefinition := range config.NodeConfig.GetStringSlice(config.CfgNetAutopeeringEntryNodes) {
-		if entryNodeDefinition == "" {
-			continue
-		}
+func parseEntryNode(entryNodeDefinition string) (entryNode *peer.Peer, err error) {
+	if entryNodeDefinition == "" {
+		return nil, nil
+	}
 
-		parts := strings.Split(entryNodeDefinition, "@")
-		if len(parts) != 2 {
-			return nil, fmt.Errorf("%w: entry node parts must be 2, is %d", ErrParsingEntryNode, len(parts))
-		}
+	parts := strings.Split(entryNodeDefinition, "@")
+	if len(parts) != 2 {
+		return nil, fmt.Errorf("%w: entry node parts must be 2, is %d", ErrParsingEntryNode, len(parts))
+	}
 
-		pubKey, err := base58.Decode(parts[0])
-		if err != nil {
-			return nil, fmt.Errorf("%w: invalid public key: %s", ErrParsingEntryNode, err)
-		}
+	pubKey, err := base58.Decode(parts[0])
+	if err != nil {
+		return nil, fmt.Errorf("%w: invalid public key: %s", ErrParsingEntryNode, err)
+	}
 
-		entryAddr, err := iputils.ParseOriginAddress(parts[1])
-		if err != nil {
-			return nil, fmt.Errorf("%w: invalid entry node address %s", err, parts[1])
-		}
+	entryAddr, err := iputils.ParseOriginAddress(parts[1])
+	if err != nil {
+		return nil, fmt.Errorf("%w: invalid entry node address %s", err, parts[1])
+	}
 
-		ipAddresses, err := iputils.GetIPAddressesFromHost(entryAddr.Addr)
-		if err != nil {
-			return nil, fmt.Errorf("%w: while handling %s", err, parts[1])
-		}
+	ipAddresses, err := iputils.GetIPAddressesFromHost(entryAddr.Addr)
+	if err != nil {
+		return nil, fmt.Errorf("%w: while handling %s", err, parts[1])
+	}
+
+	publicKey, _, err := ed25519.PublicKeyFromBytes(pubKey)
+	if err != nil {
+		return nil, err
+	}
+
+	services := service.New()
+	services.Update(service.PeeringKey, "udp", int(entryAddr.Port))
+
+	ip := ipAddresses.GetPreferredAddress(config.NodeConfig.GetBool(config.CfgNetPreferIPv6))
 
-		publicKey, _, err := ed25519.PublicKeyFromBytes(pubKey)
+	return peer.NewPeer(identity.New(publicKey), ip, services), nil
+}
+
+func parseEntryNodes() (result []*peer.Peer, err error) {
+	for _, entryNodeDefinition := range config.NodeConfig.GetStringSlice(config.CfgNetAutopeeringEntryNodes) {
+		entryNode, err := parseEntryNode(entryNodeDefinition)
 		if err != nil {
-			return nil, err
+			log.Warnf("invalid entry node; ignoring: %v, error: %v", entryNodeDefinition, err)
+			continue
 		}
+		result = append(result, entryNode)
+	}
 
-		services := service.New()
-		services.Update(service.PeeringKey, "udp", int(entryAddr.Port))
-
-		ip := ipAddresses.GetPreferredAddress(config.NodeConfig.GetBool(config.CfgNetPreferIPv6))
-		result = append(result, peer.NewPeer(identity.New(publicKey), ip, services))
+	if len(result) == 0 {
+		return nil, errors.New("no valid entry nodes found")
 	}
 
 	return result, nil

plugins/cli/cli.go

@@ -15,8 +15,13 @@ import (
 	"github.com/gohornet/hornet/pkg/config"
 )
 
-var enabledPlugins []string
-var disabledPlugins []string
+var (
+	enabledPlugins  []string
+	disabledPlugins []string
+
+	version = flag.BoolP("version", "v", false, "Prints the HORNET version")
+	help    = flag.BoolP("help", "h", false, "Prints the HORNET help")
+)
 
 func AddPluginStatus(name string, status int) {
 	switch status {
@@ -47,14 +52,22 @@ func PrintConfig() {
 	config.PrintConfig([]string{config.CfgWebAPIBasicAuthPasswordHash, config.CfgWebAPIBasicAuthPasswordSalt, config.CfgDashboardBasicAuthPasswordHash, config.CfgDashboardBasicAuthPasswordSalt})
 }
 
+// HideConfigFlags hides all non essential flags from the help/usage text.
+func HideConfigFlags() {
+	config.HideConfigFlags()
+}
+
 // PrintVersion prints out the HORNET version
 func PrintVersion() {
-	version := flag.BoolP("version", "v", false, "Prints the HORNET version")
 	flag.Parse()
 	if *version {
 		fmt.Println(AppName + " " + AppVersion)
 		os.Exit(0)
 	}
+	if *help {
+		flag.Usage()
+		os.Exit(0)
+	}
 }
 
 func printUsage() {

plugins/cli/plugin.go

@@ -21,7 +21,7 @@ import (
 
 var (
 	// AppVersion version number
-	AppVersion          = "0.4.1"
+	AppVersion          = "0.4.2"
 	LatestGithubVersion = AppVersion
 
 	// AppName app code name