Tortoise increasing memory (#1675)

## Motivation
fix tortoise increasing memory issue

cmd/node/node.go

@@ -477,7 +477,7 @@ func (app *SpacemeshApp) initServices(nodeID types.NodeId,
 	poetDb := activation.NewPoetDb(poetDbStore, app.addLogger(PoetDbLogger, lg))
 	//todo: this is initialized twice, need to refactor
 	validator := activation.NewValidator(&app.Config.POST, poetDb)
-	mdb, err := mesh.NewPersistentMeshDB(dbStorepath, app.addLogger(MeshDBLogger, lg))
+	mdb, err := mesh.NewPersistentMeshDB(dbStorepath, app.Config.BlockCacheSize, app.addLogger(MeshDBLogger, lg))
 	if err != nil {
 		return err
 	}

cmd/root.go

@@ -55,6 +55,10 @@ func AddCommands(cmd *cobra.Command) {
 
 	cmd.PersistentFlags().IntVar(&config.GenesisActiveSet, "genesis-active-size",
 		config.GenesisActiveSet, "The active set size for the genesis flow")
+
+	cmd.PersistentFlags().IntVar(&config.BlockCacheSize, "block-cache-size",
+		config.BlockCacheSize, "size in layers of meshdb block cache")
+
 	cmd.PersistentFlags().StringVar(&config.PublishEventsUrl, "events-url",
 		config.PublishEventsUrl, "publish events on this url, if no url specified event will no be published")
 

cmd/sync/sync.go

@@ -131,7 +131,7 @@ func (app *SyncApp) Start(cmd *cobra.Command, args []string) {
 
 	poetDb := activation.NewPoetDb(poetDbStore, lg.WithName("poetDb").WithOptions(log.Nop))
 
-	mshdb, err := mesh.NewPersistentMeshDB(path, lg.WithOptions(log.Nop))
+	mshdb, err := mesh.NewPersistentMeshDB(path, 5, lg.WithOptions(log.Nop))
 	if err != nil {
 		lg.Error("error: ", err)
 		return

config.toml

@@ -13,6 +13,7 @@ oracle_server = "http://localhost:3030"
 oracle_server_worldid = 0
 genesis-time = "2019-02-13T17:02:00+00:00"
 layer-duration-sec = "5"
+block-cache-size = "20"
 hdist = "5"
 coinbase = "0x1234"
 

config/config.go

@@ -95,6 +95,8 @@ type BaseConfig struct {
 	StartMining bool `mapstructure:"start-mining"`
 
 	AtxsPerBlock int `mapstructure:"atxs-per-block"`
+
+	BlockCacheSize int `mapstructure:"block-cache-size"`
 }
 
 type LoggerConfig struct {
@@ -156,6 +158,7 @@ func defaultBaseConfig() BaseConfig {
 		PoETServer:          "127.0.0.1",
 		Hdist:               5,
 		GenesisActiveSet:    5,
+		BlockCacheSize:      20,
 		SyncRequestTimeout:  2000,
 		AtxsPerBlock:        100,
 	}

mesh/meshdb.go

@@ -35,7 +35,7 @@ type MeshDB struct {
 	lhMutex            sync.Mutex
 }
 
-func NewPersistentMeshDB(path string, log log.Log) (*MeshDB, error) {
+func NewPersistentMeshDB(path string, blockCacheSize int, log log.Log) (*MeshDB, error) {
 	bdb, err := database.NewLDBDatabase(path+"blocks", 0, 0, log)
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize blocks db: %v", err)
@@ -63,7 +63,7 @@ func NewPersistentMeshDB(path string, log log.Log) (*MeshDB, error) {
 
 	ll := &MeshDB{
 		Log:                log,
-		blockCache:         NewBlockCache(100 * layerSize),
+		blockCache:         NewBlockCache(blockCacheSize * layerSize),
 		blocks:             bdb,
 		layers:             ldb,
 		transactions:       tdb,

mesh/meshdb_test.go

@@ -115,7 +115,7 @@ func createLayerWithRandVoting(index types.LayerID, prev []*types.Layer, blocksI
 }
 
 func TestForEachInView_Persistent(t *testing.T) {
-	mdb, err := NewPersistentMeshDB(Path+"/mesh_db/", log.New("TestForEachInView", "", ""))
+	mdb, err := NewPersistentMeshDB(Path+"/mesh_db/", 5, log.New("TestForEachInView", "", ""))
 	require.NoError(t, err)
 	defer mdb.Close()
 	defer teardown()
@@ -240,7 +240,7 @@ func BenchmarkNewPersistentMeshDB(b *testing.B) {
 
 	r := require.New(b)
 
-	mdb, err := NewPersistentMeshDB(path.Join(Path, "mesh_db"), log.NewDefault("meshDb"))
+	mdb, err := NewPersistentMeshDB(path.Join(Path, "mesh_db"), 5, log.NewDefault("meshDb"))
 	require.NoError(b, err)
 	defer mdb.Close()
 	defer teardown()

sync/create_baseline_test.go

@@ -33,7 +33,7 @@ func TestCreateBaseline(t *testing.T) {
 
 	id := Path
 	lg := log.New(id, "", "")
-	mshdb, _ := mesh.NewPersistentMeshDB(id, lg.WithOptions(log.Nop))
+	mshdb, _ := mesh.NewPersistentMeshDB(id, 5, lg.WithOptions(log.Nop))
 	nipstStore, _ := database.NewLDBDatabase(id+"nipst", 0, 0, lg.WithName("nipstDbStore").WithOptions(log.Nop))
 	defer nipstStore.Close()
 	atxdbStore, _ := database.NewLDBDatabase(id+"atx", 0, 0, lg.WithOptions(log.Nop))

sync/syncer_test.go

@@ -130,7 +130,7 @@ var rewardConf = mesh.Config{
 
 func getMeshWithLevelDB(id string) *mesh.Mesh {
 	lg := log.New(id, "", "")
-	mshdb, _ := mesh.NewPersistentMeshDB(id, lg)
+	mshdb, _ := mesh.NewPersistentMeshDB(id, 5, lg)
 	atxdbStore, _ := database.NewLDBDatabase(id+"atx", 0, 0, lg.WithOptions(log.Nop))
 	atxdb := activation.NewActivationDb(atxdbStore, &MockIStore{}, mshdb, 10, &ValidatorMock{}, lg.WithOptions(log.Nop))
 	return mesh.NewMesh(mshdb, atxdb, rewardConf, &MeshValidatorMock{}, &MockTxMemPool{}, &MockAtxMemPool{}, &stateMock{}, lg.WithOptions(log.Nop))

tortoise/algorithm.go

@@ -15,8 +15,6 @@ type Algorithm struct {
 type Tortoise interface {
 	handleIncomingLayer(ll *types.Layer)
 	latestComplete() types.LayerID
-	getVote(id types.BlockID) vec
-	getVotes() map[types.BlockID]vec
 }
 
 func NewAlgorithm(layerSize int, mdb *mesh.MeshDB, hdist int, lg log.Log) *Algorithm {
@@ -62,20 +60,4 @@ func (alg *Algorithm) HandleIncomingLayer(ll *types.Layer) (types.LayerID, types
 func updateMetrics(alg *Algorithm, ll *types.Layer) {
 	pbaseCount.Set(float64(alg.latestComplete()))
 	processedCount.Set(float64(ll.Index()))
-	var valid float64
-	var invalid float64
-	for _, k := range alg.getVotes() {
-		if k == Support {
-			valid++
-		} else {
-			invalid++
-		}
-	}
-	validBlocks.Set(valid)
-	invalidBlocks.Set(invalid)
-}
-
-func (alg *Algorithm) ContextualValidity(id types.BlockID) bool {
-	//todo after each layer we should persist alg.getVote(id) in mesh
-	return alg.getVote(id) == Support
 }