expose badger loading mode conf

For simplicity, I haven't exposed the entire badger config. Also, due to go default values and the badger implementation, I have had to extract the two loading mode config values so we can determine whether it has actually has been set by the user in the json config file. License: MIT Signed-off-by: Adrian Lanzafame <adrianlanzafame92@gmail.com>
ipfs-cluster · May 17, 2019 · 1ec2843 · 1ec2843
1 parent 7a66fc3
commit 1ec2843
Show file tree

Hide file tree

Showing 5 changed files with 108 additions and 7 deletions.
diff --git a/datastore/badger/badger.go b/datastore/badger/badger.go
@@ -18,8 +18,8 @@ func New(cfg *Config) (ds.Datastore, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "creating badger folder")
 	}
-	opts := &badgerds.DefaultOptions
-	return badgerds.NewDatastore(folder, opts)
+	opts := badgerds.Options{Options: cfg.BadgerOptions}
+	return badgerds.NewDatastore(folder, &opts)
 }
 
 // Cleanup deletes the badger datastore.

diff --git a/datastore/badger/config.go b/datastore/badger/config.go
@@ -5,6 +5,8 @@ import (
 	"errors"
 	"path/filepath"
 
+	"github.com/dgraph-io/badger"
+	"github.com/dgraph-io/badger/options"
 	"github.com/kelseyhightower/envconfig"
 
 	"github.com/ipfs/ipfs-cluster/config"
@@ -18,6 +20,15 @@ const (
 	DefaultSubFolder = "badger"
 )
 
+var (
+	// DefaultBadgerOptions has to be a var because badger.DefaultOptions is.
+	DefaultBadgerOptions badger.Options
+)
+
+func init() {
+	DefaultBadgerOptions = badger.DefaultOptions
+}
+
 // Config is used to initialize a BadgerDB datastore. It implements the
 // ComponentConfig interface.
 type Config struct {
@@ -26,10 +37,29 @@ type Config struct {
 	// The folder for this datastore. Non-absolute paths are relative to
 	// the base configuration folder.
 	Folder string
+
+	BadgerOptions badger.Options
 }
 
+// FileLoadingMode specifies how data in LSM table files and value log files should
+// be loaded.
+type FileLoadingMode int
+
+const (
+	// Unknown indicates that the value hasn't been set
+	Unknown FileLoadingMode = iota
+	// FileIO indicates that files must be loaded using standard I/O
+	FileIO
+	// LoadToRAM indicates that file must be loaded into RAM
+	LoadToRAM
+	// MemoryMap indicates that that the file must be memory-mapped
+	MemoryMap
+)
+
 type jsonConfig struct {
-	Folder string `json:"folder,omitempty"`
+	Folder              string          `json:"folder,omitempty"`
+	TableLoadingMode    FileLoadingMode `json:"table_loading_mode,omitempty"`     // because we can't distinguish between options.FileIO and default value
+	ValueLogLoadingMode FileLoadingMode `json:"value_log_loading_mode,omitempty"` // because we can't distinguish between options.FileIO and default value
 }
 
 // ConfigKey returns a human-friendly identifier for this type of Datastore.
@@ -40,6 +70,7 @@ func (cfg *Config) ConfigKey() string {
 // Default initializes this Config with sensible values.
 func (cfg *Config) Default() error {
 	cfg.Folder = DefaultSubFolder
+	cfg.BadgerOptions = DefaultBadgerOptions
 	return nil
 }
 
@@ -81,6 +112,12 @@ func (cfg *Config) LoadJSON(raw []byte) error {
 
 func (cfg *Config) applyJSONConfig(jcfg *jsonConfig) error {
 	config.SetIfNotDefault(jcfg.Folder, &cfg.Folder)
+	if v, ok := correctLoadingMode(jcfg.TableLoadingMode); ok {
+		cfg.BadgerOptions.TableLoadingMode = options.FileLoadingMode(v)
+	}
+	if v, ok := correctLoadingMode(jcfg.ValueLogLoadingMode); ok {
+		cfg.BadgerOptions.ValueLogLoadingMode = options.FileLoadingMode(v)
+	}
 	return cfg.Validate()
 }
 
@@ -98,6 +135,8 @@ func (cfg *Config) toJSONConfig() *jsonConfig {
 
 	if cfg.Folder != DefaultSubFolder {
 		jCfg.Folder = cfg.Folder
+		jCfg.TableLoadingMode = FileLoadingMode(cfg.BadgerOptions.TableLoadingMode)
+		jCfg.ValueLogLoadingMode = FileLoadingMode(cfg.BadgerOptions.ValueLogLoadingMode)
 	}
 
 	return jCfg
@@ -111,3 +150,12 @@ func (cfg *Config) GetFolder() string {
 
 	return filepath.Join(cfg.BaseDir, cfg.Folder)
 }
+
+func correctLoadingMode(lm FileLoadingMode) (options.FileLoadingMode, bool) {
+	switch lm {
+	default:
+		return 0, false
+	case 1, 2, 3:
+		return options.FileLoadingMode(lm - 1), true
+	}
+}
diff --git a/datastore/badger/config_test.go b/datastore/badger/config_test.go
@@ -0,0 +1,53 @@
+package badger
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/dgraph-io/badger"
+	"github.com/dgraph-io/badger/options"
+)
+
+var cfgJSON = []byte(`
+{
+    "folder": "test",
+	"value_log_loading_mode": 1
+}
+`)
+
+func TestLoadJSON(t *testing.T) {
+	cfg := &Config{}
+	err := cfg.LoadJSON(cfgJSON)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestToJSON(t *testing.T) {
+	cfg := &Config{}
+	cfg.LoadJSON(cfgJSON)
+
+	if cfg.BadgerOptions.ValueLogLoadingMode != options.FileIO {
+		t.Fatalf("got: %d, want: %d", cfg.BadgerOptions.ValueLogLoadingMode, options.FileIO)
+	}
+	if cfg.BadgerOptions.ValueLogFileSize != badger.DefaultOptions.ValueLogFileSize {
+		t.Fatalf(
+			"got: %d, want: %d",
+			cfg.BadgerOptions.ValueLogFileSize,
+			badger.DefaultOptions.ValueLogFileSize,
+		)
+	}
+
+	fmt.Printf("%+v\n", cfg.BadgerOptions)
+
+	newjson, err := cfg.ToJSON()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cfg = &Config{}
+	err = cfg.LoadJSON(newjson)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
diff --git a/go.mod b/go.mod
@@ -10,7 +10,8 @@ require (
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/btcsuite/btcd v0.0.0-20190427004231-96897255fd17 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
-	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
+	github.com/dgraph-io/badger v2.0.0-rc.2+incompatible
+	github.com/dgryski/go-farm v0.0.0-20190416075124-e1214b5e05dc // indirect
 	github.com/dustin/go-humanize v1.0.0
 	github.com/gogo/protobuf v1.2.1
 	github.com/golang/protobuf v1.3.1

diff --git a/go.sum b/go.sum
@@ -70,8 +70,8 @@ github.com/dgraph-io/badger v2.0.0-rc.2+incompatible h1:7KPp6xv5+wymkVUbkAnZZXvm
 github.com/dgraph-io/badger v2.0.0-rc.2+incompatible/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ=
 github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f h1:dDxpBYafY/GYpcl+LS4Bn3ziLPuEdGRkRjYAbSlWxSA=
 github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
-github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
-github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/dgryski/go-farm v0.0.0-20190416075124-e1214b5e05dc h1:VxEJYcOh1LMAdhIiHkofa6UC0PZvCmielUgJXgAAWFU=
+github.com/dgryski/go-farm v0.0.0-20190416075124-e1214b5e05dc/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
@@ -572,7 +572,6 @@ github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpP
 github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=
 github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
-github.com/ugorji/go v0.0.0-20171015030454-f26fc641ec9d h1:bU4z5wWQDVf3YE+OKA0wX9gNLIY4P1F0jwsCxQknJEQ=
 github.com/ugorji/go v0.0.0-20171015030454-f26fc641ec9d/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ=
 github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=