feat(dot/state): implement online pruning of historical state tries (C…

…hainSafe#1596)

chain/dev/defaults.go

@@ -39,6 +39,11 @@ var (
 	// DefaultLvl is the default log level
 	DefaultLvl = log.LvlInfo
 
+	// DefaultPruningMode is the default pruning mode
+	DefaultPruningMode = "archive"
+	// DefaultRetainBlocks is the default retained blocks
+	DefaultRetainBlocks = int64(512)
+
 	// InitConfig
 
 	// DefaultGenesis is the default genesis configuration path

chain/gssmr/defaults.go

@@ -39,6 +39,11 @@ var (
 	// DefaultLvl is the default log level
 	DefaultLvl = log.LvlInfo
 
+	// DefaultPruningMode is the default pruning mode
+	DefaultPruningMode = "archive"
+	// DefaultRetainBlocks is the default retained blocks
+	DefaultRetainBlocks = int64(512)
+
 	// InitConfig
 
 	// DefaultGenesis is the default genesis configuration path

chain/kusama/defaults.go

@@ -39,6 +39,11 @@ var (
 	// DefaultLvl is the default log level
 	DefaultLvl = log.LvlInfo
 
+	// DefaultPruningMode is the default pruning mode
+	DefaultPruningMode = "archive"
+	// DefaultRetainBlocks is the default retained blocks
+	DefaultRetainBlocks = int64(512)
+
 	// InitConfig
 
 	// DefaultGenesis is the default genesis configuration path

chain/polkadot/defaults.go

@@ -36,6 +36,11 @@ var (
 	// DefaultLvl is the default log level
 	DefaultLvl = log.LvlInfo
 
+	// DefaultPruningMode is the default pruning mode
+	DefaultPruningMode = "archive"
+	// DefaultRetainBlocks is the default pruning mode
+	DefaultRetainBlocks = int64(512)
+
 	// InitConfig
 
 	// DefaultGenesis is the default genesis configuration path

cmd/gossamer/config.go

@@ -21,10 +21,12 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/ChainSafe/gossamer/chain/dev"
 	"github.com/ChainSafe/gossamer/chain/gssmr"
 	"github.com/ChainSafe/gossamer/dot"
 	ctoml "github.com/ChainSafe/gossamer/dot/config/toml"
 	"github.com/ChainSafe/gossamer/dot/state"
+	"github.com/ChainSafe/gossamer/dot/state/pruner"
 	"github.com/ChainSafe/gossamer/dot/types"
 	"github.com/ChainSafe/gossamer/lib/common"
 	"github.com/ChainSafe/gossamer/lib/genesis"
@@ -174,6 +176,14 @@ func createInitConfig(ctx *cli.Context) (*dot.Config, error) {
 		return nil, err
 	}
 
+	if !cfg.Global.Pruning.IsValid() {
+		return nil, fmt.Errorf("--%s must be either %s or %s", PruningFlag.Name, pruner.Full, pruner.Archive)
+	}
+
+	if cfg.Global.RetainBlocks < dev.DefaultRetainBlocks {
+		return nil, fmt.Errorf("--%s cannot be less than %d", RetainBlockNumberFlag.Name, dev.DefaultRetainBlocks)
+	}
+
 	// set log config
 	err = setLogConfig(ctx, tomlCfg, &cfg.Global, &cfg.Log)
 	if err != nil {
@@ -443,6 +453,9 @@ func setDotGlobalConfigFromToml(tomlCfg *ctoml.Config, cfg *dot.GlobalConfig) {
 		}
 
 		cfg.MetricsPort = tomlCfg.Global.MetricsPort
+
+		cfg.RetainBlocks = tomlCfg.Global.RetainBlocks
+		cfg.Pruning = pruner.Mode(tomlCfg.Global.Pruning)
 	}
 }
 
@@ -472,6 +485,8 @@ func setDotGlobalConfigFromFlags(ctx *cli.Context, cfg *dot.GlobalConfig) {
 		cfg.MetricsPort = uint32(metricsPort)
 	}
 
+	cfg.RetainBlocks = ctx.Int64(RetainBlockNumberFlag.Name)
+	cfg.Pruning = pruner.Mode(ctx.String(PruningFlag.Name))
 	cfg.NoTelemetry = ctx.Bool("no-telemetry")
 }
 

cmd/gossamer/config_test.go

@@ -20,6 +20,7 @@ import (
 	"io/ioutil"
 	"testing"
 
+	"github.com/ChainSafe/gossamer/chain/dev"
 	"github.com/ChainSafe/gossamer/chain/gssmr"
 	"github.com/ChainSafe/gossamer/dot"
 	"github.com/ChainSafe/gossamer/dot/state"
@@ -45,26 +46,26 @@ func TestConfigFromChainFlag(t *testing.T) {
 	}{
 		{
 			"Test gossamer --chain gssmr",
-			[]string{"chain", "name"},
-			[]interface{}{"gssmr", dot.GssmrConfig().Global.Name},
+			[]string{"chain", "name", "pruning", "retain-blocks"},
+			[]interface{}{"gssmr", dot.GssmrConfig().Global.Name, gssmr.DefaultPruningMode, gssmr.DefaultRetainBlocks},
 			dot.GssmrConfig(),
 		},
 		{
 			"Test gossamer --chain kusama",
-			[]string{"chain", "name"},
-			[]interface{}{"kusama", dot.KusamaConfig().Global.Name},
+			[]string{"chain", "name", "pruning", "retain-blocks"},
+			[]interface{}{"kusama", dot.KusamaConfig().Global.Name, gssmr.DefaultPruningMode, gssmr.DefaultRetainBlocks},
 			dot.KusamaConfig(),
 		},
 		{
 			"Test gossamer --chain polkadot",
-			[]string{"chain", "name"},
-			[]interface{}{"polkadot", dot.PolkadotConfig().Global.Name},
+			[]string{"chain", "name", "pruning", "retain-blocks"},
+			[]interface{}{"polkadot", dot.PolkadotConfig().Global.Name, gssmr.DefaultPruningMode, gssmr.DefaultRetainBlocks},
 			dot.PolkadotConfig(),
 		},
 		{
 			"Test gossamer --chain dev",
-			[]string{"chain", "name"},
-			[]interface{}{"dev", dot.DevConfig().Global.Name},
+			[]string{"chain", "name", "pruning", "retain-blocks"},
+			[]interface{}{"dev", dot.DevConfig().Global.Name, dev.DefaultPruningMode, dev.DefaultRetainBlocks},
 			dot.DevConfig(),
 		},
 	}
@@ -100,8 +101,8 @@ func TestInitConfigFromFlags(t *testing.T) {
 	}{
 		{
 			"Test gossamer --genesis",
-			[]string{"config", "genesis"},
-			[]interface{}{testCfgFile.Name(), "test_genesis"},
+			[]string{"config", "genesis", "pruning", "retain-blocks"},
+			[]interface{}{testCfgFile.Name(), "test_genesis", dev.DefaultPruningMode, dev.DefaultRetainBlocks},
 			dot.InitConfig{
 				Genesis: "test_genesis",
 			},

cmd/gossamer/export.go

@@ -79,11 +79,13 @@ func dotConfigToToml(dcfg *dot.Config) *ctoml.Config {
 	cfg := &ctoml.Config{}
 
 	cfg.Global = ctoml.GlobalConfig{
-		Name:        dcfg.Global.Name,
-		ID:          dcfg.Global.ID,
-		BasePath:    dcfg.Global.BasePath,
-		LogLvl:      dcfg.Global.LogLvl.String(),
-		MetricsPort: dcfg.Global.MetricsPort,
+		Name:         dcfg.Global.Name,
+		ID:           dcfg.Global.ID,
+		BasePath:     dcfg.Global.BasePath,
+		LogLvl:       dcfg.Global.LogLvl.String(),
+		MetricsPort:  dcfg.Global.MetricsPort,
+		RetainBlocks: dcfg.Global.RetainBlocks,
+		Pruning:      string(dcfg.Global.Pruning),
 	}
 
 	cfg.Log = ctoml.LogConfig{

cmd/gossamer/export_test.go

@@ -20,6 +20,7 @@ import (
 	"io/ioutil"
 	"testing"
 
+	"github.com/ChainSafe/gossamer/chain/gssmr"
 	"github.com/ChainSafe/gossamer/dot"
 	"github.com/ChainSafe/gossamer/lib/utils"
 
@@ -91,8 +92,8 @@ func TestExportCommand(t *testing.T) {
 		},
 		{
 			"Test gossamer export --config --genesis --bootnodes --log --force",
-			[]string{"config", "genesis", "bootnodes", "name", "force"},
-			[]interface{}{testConfig, genFile.Name(), testBootnode, "Gossamer", "true"},
+			[]string{"config", "genesis", "bootnodes", "name", "force", "pruning", "retain-blocks"},
+			[]interface{}{testConfig, genFile.Name(), testBootnode, "Gossamer", "true", gssmr.DefaultPruningMode, gssmr.DefaultRetainBlocks},
 			&dot.Config{
 				Global: testCfg.Global,
 				Init: dot.InitConfig{
@@ -122,8 +123,8 @@ func TestExportCommand(t *testing.T) {
 		},
 		{
 			"Test gossamer export --config --genesis --protocol --log --force",
-			[]string{"config", "genesis", "protocol", "force", "name"},
-			[]interface{}{testConfig, genFile.Name(), testProtocol, "true", "Gossamer"},
+			[]string{"config", "genesis", "protocol", "force", "name", "pruning", "retain-blocks"},
+			[]interface{}{testConfig, genFile.Name(), testProtocol, "true", "Gossamer", gssmr.DefaultPruningMode, gssmr.DefaultRetainBlocks},
 			&dot.Config{
 				Global: testCfg.Global,
 				Init: dot.InitConfig{

cmd/gossamer/flags.go

@@ -17,6 +17,7 @@
 package main
 
 import (
+	"github.com/ChainSafe/gossamer/chain/dev"
 	log "github.com/ChainSafe/log15"
 	"github.com/urfave/cli"
 )
@@ -280,10 +281,18 @@ var (
 	}
 
 	// RetainBlockNumberFlag retain number of block from latest block while pruning, valid for the use with prune-state subcommand
-	RetainBlockNumberFlag = cli.IntFlag{
+	RetainBlockNumberFlag = cli.Int64Flag{
 		Name:  "retain-blocks",
 		Usage: "Retain number of block from latest block while pruning",
-		Value: 256,
+		Value: dev.DefaultRetainBlocks,
+	}
+
+	// PruningFlag triggers the online pruning of historical state tries. It's either full or archive. To enable pruning the value
+	// should be set to `full`.
+	PruningFlag = cli.StringFlag{
+		Name:  "pruning",
+		Usage: `State trie online pruning ("full", "archive")`,
+		Value: dev.DefaultPruningMode,
 	}
 )
 
@@ -301,7 +310,6 @@ var (
 		RewindFlag,
 		DBPathFlag,
 		BloomFilterSizeFlag,
-		RetainBlockNumberFlag,
 	}
 
 	// StartupFlags are flags that are valid for use with the root command and the export subcommand
@@ -346,6 +354,8 @@ var (
 	InitFlags = append([]cli.Flag{
 		ForceFlag,
 		GenesisFlag,
+		PruningFlag,
+		RetainBlockNumberFlag,
 	}, GlobalFlags...)
 
 	BuildSpecFlags = append([]cli.Flag{

cmd/gossamer/flags_test.go

@@ -20,6 +20,7 @@ import (
 	"io/ioutil"
 	"testing"
 
+	"github.com/ChainSafe/gossamer/chain/dev"
 	"github.com/ChainSafe/gossamer/dot"
 	"github.com/ChainSafe/gossamer/lib/utils"
 
@@ -43,24 +44,24 @@ func TestFixFlagOrder(t *testing.T) {
 		values      []interface{}
 	}{
 		{
-			"Test gossamer --config --genesis --log --force",
-			[]string{"config", "genesis", "log", "force"},
-			[]interface{}{testConfig.Name(), genFile.Name(), "trace", true},
+			"Test gossamer --config --genesis --log --force --pruning --retain-blocks",
+			[]string{"config", "genesis", "log", "force", "pruning", "retain-blocks"},
+			[]interface{}{testConfig.Name(), genFile.Name(), "trace", true, dev.DefaultPruningMode, dev.DefaultRetainBlocks},
 		},
 		{
-			"Test gossamer --config --genesis --force --log",
-			[]string{"config", "genesis", "force", "log"},
-			[]interface{}{testConfig.Name(), genFile.Name(), true, "trace"},
+			"Test gossamer --config --genesis --force --log --pruning --retain-blocks",
+			[]string{"config", "genesis", "force", "log", "pruning", "retain-blocks"},
+			[]interface{}{testConfig.Name(), genFile.Name(), true, "trace", dev.DefaultPruningMode, dev.DefaultRetainBlocks},
 		},
 		{
-			"Test gossamer --config --force --genesis --log",
-			[]string{"config", "force", "genesis", "log"},
-			[]interface{}{testConfig.Name(), true, genFile.Name(), "trace"},
+			"Test gossamer --config --force --genesis --log ---pruning --retain-blocks",
+			[]string{"config", "force", "genesis", "log", "pruning", "retain-blocks"},
+			[]interface{}{testConfig.Name(), true, genFile.Name(), "trace", dev.DefaultPruningMode, dev.DefaultRetainBlocks},
 		},
 		{
-			"Test gossamer --force --config --genesis --log",
-			[]string{"force", "config", "genesis", "log"},
-			[]interface{}{true, testConfig.Name(), genFile.Name(), "trace"},
+			"Test gossamer --force --config --genesis --log --pruning --retain-blocks",
+			[]string{"force", "config", "genesis", "log", "pruning", "retain-blocks"},
+			[]interface{}{true, testConfig.Name(), genFile.Name(), "trace", dev.DefaultPruningMode, dev.DefaultRetainBlocks},
 		},
 	}
 

cmd/gossamer/main.go

@@ -333,7 +333,6 @@ func initAction(ctx *cli.Context) error {
 	// expand data directory and update node configuration (performed separately
 	// from createDotConfig because dot config should not include expanded path)
 	cfg.Global.BasePath = utils.ExpandDir(cfg.Global.BasePath)
-
 	// check if node has been initialised (expected false - no warning log)
 	if dot.NodeInitialized(cfg.Global.BasePath, false) {
 
@@ -443,12 +442,12 @@ func pruneState(ctx *cli.Context) error {
 	bloomSize := ctx.GlobalUint64(BloomFilterSizeFlag.Name)
 	retainBlocks := ctx.GlobalInt64(RetainBlockNumberFlag.Name)
 
-	pruner, err := state.NewPruner(inputDBPath, prunedDBPath, bloomSize, retainBlocks)
+	pruner, err := state.NewOfflinePruner(inputDBPath, prunedDBPath, bloomSize, retainBlocks)
 	if err != nil {
 		return err
 	}
 
-	logger.Info("Pruner initialised")
+	logger.Info("Offline pruner initialised")
 
 	err = pruner.SetBloomFilter()
 	if err != nil {

cmd/gossamer/prune_test.go

@@ -22,7 +22,7 @@ func runPruneCmd(t *testing.T, configFile, prunedDBPath string) {
 	ctx, err := newTestContext(
 		"Test state trie offline pruning  --prune-state",
 		[]string{"config", "pruned-db-path", "bloom-size", "retain-blocks"},
-		[]interface{}{configFile, prunedDBPath, "256", "5"},
+		[]interface{}{configFile, prunedDBPath, "256", int64(5)},
 	)
 	if err != nil {
 		t.Fatal(err)

cmd/gossamer/utils.go

@@ -95,6 +95,8 @@ func newTestConfig(t *testing.T) *dot.Config {
 			LogLvl:         log.LvlInfo,
 			PublishMetrics: dot.GssmrConfig().Global.PublishMetrics,
 			MetricsPort:    dot.GssmrConfig().Global.MetricsPort,
+			RetainBlocks:   dot.GssmrConfig().Global.RetainBlocks,
+			Pruning:        dot.GssmrConfig().Global.Pruning,
 		},
 		Log: dot.LogConfig{
 			CoreLvl:           log.LvlInfo,

cmd/gossamer/utils_test.go

@@ -38,6 +38,8 @@ func newTestContext(description string, flags []string, values []interface{}) (*
 			set.String(flags[i], v, "")
 		case uint:
 			set.Uint(flags[i], v, "")
+		case int64:
+			set.Int64(flags[i], v, "")
 		default:
 			return nil, fmt.Errorf("unexpected cli value type: %T", values[i])
 		}
@@ -69,6 +71,11 @@ func newTestContext(description string, flags []string, values []interface{}) (*
 			if err != nil {
 				return nil, fmt.Errorf("failed to set cli flag: %T", flags[i])
 			}
+		case int64:
+			err := ctx.Set(flags[i], strconv.Itoa(int(values[i].(int64))))
+			if err != nil {
+				return nil, fmt.Errorf("failed to set cli flag: %T", flags[i])
+			}
 		default:
 			return nil, fmt.Errorf("unexpected cli value type: %T", values[i])
 		}

dot/build_spec.go

@@ -107,7 +107,11 @@ func BuildFromDB(path string) (*BuildSpec, error) {
 	tmpGen.Genesis.Raw = make(map[string]map[string]string)
 	tmpGen.Genesis.Runtime = make(map[string]map[string]interface{})
 
-	stateSrvc := state.NewService(path, log.LvlCrit)
+	config := state.Config{
+		Path:     path,
+		LogLevel: log.LvlInfo,
+	}
+	stateSrvc := state.NewService(config)
 
 	// start state service (initialise state database)
 	err := stateSrvc.Start()