Merge pull request #29 from gohornet/add-profiles

Add profiles

CHANGELOG.md

@@ -2,6 +2,25 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.2.5] - 15.12.2019
+
+### Added
+
+    - More badger options in the profiles
+    - "auto" profile chooses best setting based on available system memory
+
+### Changed
+
+    - "compactLevel0OnClose" is now disabled per default
+    - Faster shutdown of the node
+
+### Config file changes
+
+New option:
+```json
+  "useProfile": "auto",
+```
+
 ## [0.2.4] - 15.12.2019
 
 This release fixes a CRITICAL bug! You have to delete your database folder.

config.json

@@ -1,5 +1,5 @@
 {
-  "useProfile": "default",
+  "useProfile": "auto",
   "profiles": {
     "custom": {
       "caches": {
@@ -44,14 +44,21 @@
         "numLevelZeroTables": 5,
         "numLevelZeroTablesStall": 10,
         "numMemtables": 5,
+        "bloomFalsePositive": 0.01,
+        "blockSize": 4096,
+        "syncWrites": false,
         "numVersionsToKeep": 1,
-        "syncWrites": true,
-        "compactLevel0OnClose": true,
+        "compactLevel0OnClose": false,
+        "keepL0InMemory": false,
+        "verifyValueChecksum": false,
+        "maxCacheSize": 50000000,
+        "ZSTDCompressionLevel": 10,
         "valueLogFileSize": 1073741823,
         "valueLogMaxEntries": 1000000,
         "valueThreshold": 32,
+        "withTruncate": false,
         "logRotatesToFlush": 2,
-        "maxCacheSize": 50000000
+        "eventLogging": false
       }
     }
   },

go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/labstack/echo v3.3.10+incompatible
 	github.com/mitchellh/mapstructure v1.1.2
 	github.com/pkg/errors v0.8.1
+	github.com/shirou/gopsutil v2.19.11+incompatible
 	github.com/spf13/pflag v1.0.5
 	github.com/valyala/fasttemplate v1.1.0 // indirect
 	go.uber.org/atomic v1.5.1

go.sum

@@ -242,6 +242,8 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR
 github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y=
 github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shirou/gopsutil v2.19.11+incompatible h1:lJHR0foqAjI4exXqWsU3DbH7bX1xvdhGdnXTIARA9W4=
+github.com/shirou/gopsutil v2.19.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY=
 github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM=
 github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0=

packages/database/badger_instance.go

@@ -65,14 +65,21 @@ func createDB() (*badger.DB, error) {
 		WithNumLevelZeroTables(badgerOpts.NumLevelZeroTables).
 		WithNumLevelZeroTablesStall(badgerOpts.NumLevelZeroTablesStall).
 		WithNumMemtables(badgerOpts.NumMemtables).
+		WithBloomFalsePositive(badgerOpts.BloomFalsePositive).
+		WithBlockSize(badgerOpts.BlockSize).
 		WithSyncWrites(badgerOpts.SyncWrites).
 		WithNumVersionsToKeep(badgerOpts.NumVersionsToKeep).
 		WithCompactL0OnClose(badgerOpts.CompactLevel0OnClose).
+		WithKeepL0InMemory(badgerOpts.KeepL0InMemory).
+		WithVerifyValueChecksum(badgerOpts.VerifyValueChecksum).
+		WithMaxCacheSize(badgerOpts.MaxCacheSize).
+		WithZSTDCompressionLevel(badgerOpts.ZSTDCompressionLevel).
 		WithValueLogFileSize(badgerOpts.ValueLogFileSize).
 		WithValueLogMaxEntries(badgerOpts.ValueLogMaxEntries).
 		WithValueThreshold(badgerOpts.ValueThreshold).
+		WithTruncate(badgerOpts.WithTruncate).
 		WithLogRotatesToFlush(badgerOpts.LogRotatesToFlush).
-		WithMaxCacheSize(badgerOpts.MaxCacheSize)
+		WithEventLogging(badgerOpts.EventLogging)
 
 	if runtime.GOOS == "windows" {
 		opts = opts.WithTruncate(true)

packages/profile/profile.go

@@ -1,26 +1,59 @@
 package profile
 
 import (
+	"errors"
 	"fmt"
 	"sync"
 
 	"github.com/dgraph-io/badger/v2/options"
 	"github.com/iotaledger/hive.go/parameter"
+	"github.com/shirou/gopsutil/mem"
 )
 
-var once = sync.Once{}
-var profile *Profile
+var (
+	once    = sync.Once{}
+	profile *Profile
+
+	ErrNotEnoughMemory = errors.New("Not enough system memory")
+)
 
 func GetProfile() *Profile {
 	once.Do(func() {
 		profileName := parameter.NodeConfig.GetString("useProfile")
+		if profileName == "auto" {
+			v, err := mem.VirtualMemory()
+			if err != nil {
+				panic(err)
+			}
+
+			if v.Total >= 8000000000 {
+				profileName = "8gb"
+			} else if v.Total >= 4000000000 {
+				profileName = "4gb"
+			} else if v.Total >= 2000000000 {
+				profileName = "2gb"
+			} else if v.Total >= 1000000000 {
+				profileName = "1gb"
+			} else {
+				panic(ErrNotEnoughMemory)
+			}
+
+			println(fmt.Sprintf("Profile mode 'auto', Available system memory: %d bytes, Used profile: %s", v.Total, profileName))
+		}
+
 		switch profileName {
-		case "default":
-			profile = DefaultProfile
-			profile.Name = "default"
-		case "light":
-			profile = LightProfile
-			profile.Name = "light"
+		case "8gb", "default":
+			profile = Profile8GB
+			profile.Name = "8gb"
+		case "4gb":
+			profile = Profile4GB
+			profile.Name = "4gb"
+		case "2gb":
+			profile = Profile2GB
+			profile.Name = "2gb"
+		case "1gb", "light":
+			profile = Profile1GB
+			profile.Name = "1gb"
 		default:
 			p := &Profile{}
 			key := fmt.Sprintf("profiles.%s", profileName)
@@ -37,7 +70,7 @@ func GetProfile() *Profile {
 	return profile
 }
 
-var DefaultProfile = &Profile{
+var Profile8GB = &Profile{
 	Caches: Caches{
 		RequestQueue: CacheOpts{
 			Size: 100000,
@@ -80,18 +113,147 @@ var DefaultProfile = &Profile{
 		NumLevelZeroTables:      5,
 		NumLevelZeroTablesStall: 10,
 		NumMemtables:            5,
+		BloomFalsePositive:      0.01,
+		BlockSize:               4 * 1024,
+		SyncWrites:              false,
 		NumVersionsToKeep:       1,
-		SyncWrites:              true,
-		CompactLevel0OnClose:    true,
+		CompactLevel0OnClose:    false,
+		KeepL0InMemory:          false,
+		VerifyValueChecksum:     false,
+		MaxCacheSize:            50000000,
+		ZSTDCompressionLevel:    10,
 		ValueLogFileSize:        1073741823,
 		ValueLogMaxEntries:      1000000,
 		ValueThreshold:          32,
+		WithTruncate:            false,
 		LogRotatesToFlush:       2,
+		EventLogging:            false,
+	},
+}
+
+var Profile4GB = &Profile{
+	Caches: Caches{
+		RequestQueue: CacheOpts{
+			Size: 100000,
+		},
+		Approvers: CacheOpts{
+			Size:         100000,
+			EvictionSize: 1000,
+		},
+		Bundles: CacheOpts{
+			Size:         20000,
+			EvictionSize: 1000,
+		},
+		Milestones: CacheOpts{
+			Size:         1000,
+			EvictionSize: 100,
+		},
+		SpentAddresses: CacheOpts{
+			Size:         5000,
+			EvictionSize: 1000,
+		},
+		Transactions: CacheOpts{
+			Size:         50000,
+			EvictionSize: 1000,
+		},
+		IncomingTransactionFilter: CacheOpts{
+			Size: 5000,
+		},
+		RefsInvalidBundle: CacheOpts{
+			Size: 10000,
+		},
+	},
+	Badger: BadgerOpts{
+		LevelOneSize:            268435456,
+		LevelSizeMultiplier:     10,
+		TableLoadingMode:        options.FileIO,
+		ValueLogLoadingMode:     options.FileIO,
+		MaxLevels:               7,
+		MaxTableSize:            67108864,
+		NumCompactors:           2,
+		NumLevelZeroTables:      5,
+		NumLevelZeroTablesStall: 10,
+		NumMemtables:            5,
+		BloomFalsePositive:      0.01,
+		BlockSize:               4 * 1024,
+		SyncWrites:              false,
+		NumVersionsToKeep:       1,
+		CompactLevel0OnClose:    false,
+		KeepL0InMemory:          false,
+		VerifyValueChecksum:     false,
 		MaxCacheSize:            50000000,
+		ZSTDCompressionLevel:    10,
+		ValueLogFileSize:        1073741823,
+		ValueLogMaxEntries:      1000000,
+		ValueThreshold:          32,
+		WithTruncate:            false,
+		LogRotatesToFlush:       2,
+		EventLogging:            false,
+	},
+}
+
+var Profile2GB = &Profile{
+	Caches: Caches{
+		RequestQueue: CacheOpts{
+			Size: 100000,
+		},
+		Approvers: CacheOpts{
+			Size:         50000,
+			EvictionSize: 1000,
+		},
+		Bundles: CacheOpts{
+			Size:         10000,
+			EvictionSize: 1000,
+		},
+		Milestones: CacheOpts{
+			Size:         1000,
+			EvictionSize: 100,
+		},
+		SpentAddresses: CacheOpts{
+			Size:         2000,
+			EvictionSize: 1000,
+		},
+		Transactions: CacheOpts{
+			Size:         25000,
+			EvictionSize: 1000,
+		},
+		IncomingTransactionFilter: CacheOpts{
+			Size: 5000,
+		},
+		RefsInvalidBundle: CacheOpts{
+			Size: 10000,
+		},
+	},
+	Badger: BadgerOpts{
+		LevelOneSize:            268435456,
+		LevelSizeMultiplier:     10,
+		TableLoadingMode:        options.FileIO,
+		ValueLogLoadingMode:     options.FileIO,
+		MaxLevels:               7,
+		MaxTableSize:            67108864,
+		NumCompactors:           2,
+		NumLevelZeroTables:      5,
+		NumLevelZeroTablesStall: 10,
+		NumMemtables:            5,
+		BloomFalsePositive:      0.01,
+		BlockSize:               4 * 1024,
+		SyncWrites:              false,
+		NumVersionsToKeep:       1,
+		CompactLevel0OnClose:    false,
+		KeepL0InMemory:          false,
+		VerifyValueChecksum:     false,
+		MaxCacheSize:            50000000,
+		ZSTDCompressionLevel:    10,
+		ValueLogFileSize:        1073741823,
+		ValueLogMaxEntries:      1000000,
+		ValueThreshold:          32,
+		WithTruncate:            false,
+		LogRotatesToFlush:       2,
+		EventLogging:            false,
 	},
 }
 
-var LightProfile = &Profile{
+var Profile1GB = &Profile{
 	Caches: Caches{
 		RequestQueue: CacheOpts{
 			Size: 100000,
@@ -134,14 +296,21 @@ var LightProfile = &Profile{
 		NumLevelZeroTables:      1,
 		NumLevelZeroTablesStall: 2,
 		NumMemtables:            1,
-		NumVersionsToKeep:       1,
+		BloomFalsePositive:      0.01,
+		BlockSize:               4 * 1024,
 		SyncWrites:              false,
-		CompactLevel0OnClose:    true,
+		NumVersionsToKeep:       1,
+		CompactLevel0OnClose:    false,
+		KeepL0InMemory:          false,
+		VerifyValueChecksum:     false,
+		MaxCacheSize:            50000000,
+		ZSTDCompressionLevel:    10,
 		ValueLogFileSize:        33554431,
 		ValueLogMaxEntries:      250000,
 		ValueThreshold:          32,
+		WithTruncate:            false,
 		LogRotatesToFlush:       2,
-		MaxCacheSize:            50000000,
+		EventLogging:            false,
 	},
 }
 
@@ -178,12 +347,19 @@ type BadgerOpts struct {
 	NumLevelZeroTables      int                     `json:"numLevelZeroTables"`
 	NumLevelZeroTablesStall int                     `json:"numLevelZeroTablesStall"`
 	NumMemtables            int                     `json:"numMemtables"`
-	NumVersionsToKeep       int                     `json:"numVersionsToKeep"`
+	BloomFalsePositive      float64                 `json:"bloomFalsePositive"`
+	BlockSize               int                     `json:"blockSize"`
 	SyncWrites              bool                    `json:"syncWrites"`
+	NumVersionsToKeep       int                     `json:"numVersionsToKeep"`
 	CompactLevel0OnClose    bool                    `json:"compactLevel0OnClose"`
+	KeepL0InMemory          bool                    `json:"keepL0InMemory"`
+	VerifyValueChecksum     bool                    `json:"verifyValueChecksum"`
+	MaxCacheSize            int64                   `json:"maxCacheSize"`
+	ZSTDCompressionLevel    int                     `json:"ZSTDCompressionLevel"`
 	ValueLogFileSize        int64                   `json:"valueLogFileSize"`
 	ValueLogMaxEntries      uint32                  `json:"valueLogMaxEntries"`
 	ValueThreshold          int                     `json:"valueThreshold"`
+	WithTruncate            bool                    `json:"withTruncate"`
 	LogRotatesToFlush       int32                   `json:"logRotatesToFlush"`
-	MaxCacheSize            int64                   `json:"maxCacheSize"`
+	EventLogging            bool                    `json:"eventLogging"`
 }

plugins/cli/plugin.go

@@ -13,7 +13,7 @@ import (
 
 var (
 	// AppVersion version number
-	AppVersion = "0.2.4"
+	AppVersion = "0.2.5"
 
 	// AppName app code name
 	AppName = "HORNET"