Rectify DB debug APIs chaindbProperty and chaindbCompact

api/api_private_debug.go

@@ -24,13 +24,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"strings"
+	"time"
 
 	"github.com/davecgh/go-spew/spew"
+	"github.com/klaytn/klaytn/common"
 	"github.com/klaytn/klaytn/networks/rpc"
-	"github.com/klaytn/klaytn/storage/database"
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
 // PrivateDebugAPI is the collection of Klaytn APIs exposed over the private
@@ -45,42 +43,29 @@ func NewPrivateDebugAPI(b Backend) *PrivateDebugAPI {
 	return &PrivateDebugAPI{b: b}
 }
 
-// GetDBProperty returns the value of the given property of the given database.
-func (api *PrivateDebugAPI) GetDBProperty(dt database.DBEntryType, name string) string {
-	return api.b.ChainDB().GetProperty(dt, name)
-}
-
 // ChaindbProperty returns leveldb properties of the chain database.
 func (api *PrivateDebugAPI) ChaindbProperty(property string) (string, error) {
-	ldb, ok := api.b.ChainDB().(interface {
-		LDB() *leveldb.DB
-	})
-	if !ok {
-		return "", fmt.Errorf("chaindbProperty does not work for memory databases")
-	}
-	if property == "" {
-		property = "leveldb.stats"
-	} else if !strings.HasPrefix(property, "leveldb.") {
-		property = "leveldb." + property
-	}
-	return ldb.LDB().GetProperty(property)
+	return api.b.ChainDB().Stat(property)
 }
 
-// ChaindbCompact compacts the chain database if successful, otherwise it returns nil.
+// ChaindbCompact flattens the entire key-value database into a single level,
+// removing all unused slots and merging all keys.
 func (api *PrivateDebugAPI) ChaindbCompact() error {
-	ldb, ok := api.b.ChainDB().(interface {
-		LDB() *leveldb.DB
-	})
-	if !ok {
-		return fmt.Errorf("chaindbCompact does not work for memory databases")
-	}
-	for b := byte(0); b < 255; b++ {
-		logger.Info("Compacting chain database", "range", fmt.Sprintf("0x%0.2X-0x%0.2X", b, b+1))
-		err := ldb.LDB().CompactRange(util.Range{Start: []byte{b}, Limit: []byte{b + 1}})
-		if err != nil {
+	for b := 0; b <= 255; b++ {
+		var (
+			start = []byte{byte(b)}
+			end   = []byte{byte(b + 1)}
+		)
+		if b == 255 {
+			end = nil
+		}
+		logger.Info("Compacting database started", "range", fmt.Sprintf("%#X-%#X", start, end))
+		cstart := time.Now()
+		if err := api.b.ChainDB().Compact(start, end); err != nil {
 			logger.Error("Database compaction failed", "err", err)
 			return err
 		}
+		logger.Info("Compacting database completed", "range", fmt.Sprintf("%#X-%#X", start, end), "elapsed", common.PrettyDuration(time.Since(cstart)))
 	}
 	return nil
 }

console/web3ext/web3ext.go

@@ -600,12 +600,6 @@ web3._extend({
 			call: 'debug_startCollectingTrieStats',
 			params: 1,
 		}),
-		new web3._extend.Method({
-			name: 'getDBProperty',
-			call: 'debug_getDBProperty',
-			params: 2,
-			outputFormatter: console.log
-		}),
 		new web3._extend.Method({
 			name: 'chaindbProperty',
 			call: 'debug_chaindbProperty',

storage/database/badger_database.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/dgraph-io/badger"
 	"github.com/klaytn/klaytn/log"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -202,10 +203,6 @@ func (bg *badgerDB) Meter(prefix string) {
 	logger.Warn("badgerDB does not support metrics!")
 }
 
-func (bg *badgerDB) GetProperty(name string) string {
-	return ""
-}
-
 func (bg *badgerDB) TryCatchUpWithPrimary() error {
 	return nil
 }
@@ -257,3 +254,11 @@ func (b *badgerBatch) Replay(w KeyValueWriter) error {
 	logger.CritWithStack("Replay is not implemented in badgerBatch!")
 	return nil
 }
+
+func (bg *badgerDB) Stat(property string) (string, error) {
+	return "", errors.New("unknown property")
+}
+
+func (bg *badgerDB) Compact(start []byte, limit []byte) error {
+	return nil
+}

storage/database/db_manager.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"encoding/binary"
 	"encoding/json"
+	"fmt"
 	"math/big"
 	"os"
 	"path/filepath"
@@ -68,7 +69,6 @@ type DBManager interface {
 	GetStateTrieMigrationDB() Database
 	GetMiscDB() Database
 	GetSnapshotDB() Database
-	GetProperty(dt DBEntryType, name string) string
 
 	// from accessors_chain.go
 	ReadCanonicalHash(number uint64) common.Hash
@@ -306,6 +306,9 @@ type DBManager interface {
 	ReadChainDataFetcherCheckpoint() (uint64, error)
 
 	TryCatchUpWithPrimary() error
+
+	Stat(string) (string, error)
+	Compact([]byte, []byte) error
 }
 
 type DBEntryType uint8
@@ -867,10 +870,6 @@ func (dbm *databaseManager) GetSnapshotDB() Database {
 	return dbm.getDatabase(SnapshotDB)
 }
 
-func (dbm *databaseManager) GetProperty(dt DBEntryType, name string) string {
-	return dbm.getDatabase(dt).GetProperty(name)
-}
-
 func (dbm *databaseManager) TryCatchUpWithPrimary() error {
 	for _, db := range dbm.dbs {
 		if db != nil {
@@ -882,6 +881,44 @@ func (dbm *databaseManager) TryCatchUpWithPrimary() error {
 	return nil
 }
 
+func (dbm *databaseManager) Stat(property string) (string, error) {
+	stats := ""
+	errs := ""
+	for idx, db := range dbm.dbs {
+		if db != nil {
+			stat, err := db.Stat(property)
+			headInfo := fmt.Sprintf(" [%s:%s]\n", DBEntryType(idx), db.Type())
+			if err == nil {
+				stats += headInfo + stat
+			} else {
+				errs += headInfo + err.Error()
+			}
+		}
+	}
+	if errs == "" {
+		return stats, nil
+	} else {
+		return stats, errors.New(errs)
+	}
+}
+
+func (dbm *databaseManager) Compact(start []byte, limit []byte) error {
+	errs := ""
+	for idx, db := range dbm.dbs {
+		if db != nil {
+			if err := db.Compact(start, limit); err != nil {
+				headInfo := fmt.Sprintf(" [%s:%s]\n", DBEntryType(idx), db.Type())
+				errs = headInfo + err.Error()
+			}
+		}
+	}
+	if errs == "" {
+		return nil
+	} else {
+		return errors.New(errs)
+	}
+}
+
 func (dbm *databaseManager) GetMemDB() *MemDB {
 	if dbm.config.DBType == MemoryDB {
 		if memDB, ok := dbm.dbs[0].(*MemDB); ok {

storage/database/dynamodb.go

@@ -69,8 +69,10 @@ const (
 )
 
 // batch write
-const WorkerNum = 10
-const itemChanSize = WorkerNum * 2
+const (
+	WorkerNum    = 10
+	itemChanSize = WorkerNum * 2
+)
 
 var (
 	dynamoDBClient    *dynamodb.DynamoDB          // handles dynamoDB connections
@@ -444,10 +446,6 @@ func (dynamo *dynamoDB) Meter(prefix string) {
 	dynamoBatchWriteTimeMeter = metrics.NewRegisteredMeter(prefix+"batchwrite/time", nil)
 }
 
-func (dynamo *dynamoDB) GetProperty(name string) string {
-	return ""
-}
-
 func (dynamo *dynamoDB) TryCatchUpWithPrimary() error {
 	return nil
 }
@@ -629,3 +627,11 @@ func (batch *dynamoBatch) Replay(w KeyValueWriter) error {
 	logger.CritWithStack("Replay should not be called when using dynamodb batch")
 	return nil
 }
+
+func (dynamo *dynamoDB) Stat(property string) (string, error) {
+	return "", errors.New("unknown property")
+}
+
+func (dynamo *dynamoDB) Compact(start []byte, limit []byte) error {
+	return nil
+}

storage/database/interface.go

@@ -68,9 +68,30 @@ type KeyValueWriter interface {
 	Delete(key []byte) error
 }
 
+// KeyValueStater wraps the Stat method of a backing data store.
+type KeyValueStater interface {
+	// Stat returns a particular internal stat of the database.
+	Stat(property string) (string, error)
+}
+
+// Compacter wraps the Compact method of a backing data store.
+type Compacter interface {
+	// Compact flattens the underlying data store for the given key range. In essence,
+	// deleted and overwritten versions are discarded, and the data is rearranged to
+	// reduce the cost of operations needed to access them.
+	//
+	// A nil start is treated as a key before all keys in the data store; a nil limit
+	// is treated as a key after all keys in the data store. If both is nil then it
+	// will compact entire data store.
+	Compact(start []byte, limit []byte) error
+}
+
 // Database wraps all database operations. All methods are safe for concurrent use.
 type Database interface {
 	KeyValueWriter
+	KeyValueStater
+	Compacter
+
 	Get(key []byte) ([]byte, error)
 	Has(key []byte) (bool, error)
 	Close()
@@ -79,7 +100,6 @@ type Database interface {
 	Meter(prefix string)
 	Iteratee
 
-	GetProperty(name string) string
 	TryCatchUpWithPrimary() error
 }
 

storage/database/leveldb_database.go

@@ -22,6 +22,7 @@ package database
 
 import (
 	"fmt"
+	"strings"
 	"sync"
 	"time"
 
@@ -340,8 +341,17 @@ func (db *levelDB) Close() {
 	}
 }
 
-func (db *levelDB) LDB() *leveldb.DB {
-	return db.db
+func (db *levelDB) Stat(property string) (string, error) {
+	if property == "" {
+		property = "leveldb.stats"
+	} else if !strings.HasPrefix(property, "leveldb.") {
+		property = "leveldb." + property
+	}
+	return db.db.GetProperty(property)
+}
+
+func (db *levelDB) Compact(start []byte, limit []byte) error {
+	return db.db.CompactRange(util.Range{Start: start, Limit: limit})
 }
 
 // Meter configures the database metrics collectors and
@@ -504,10 +514,6 @@ func (db *levelDB) updateLevelStats(s *leveldb.DBStats, lv int) {
 	db.levelDurationsGauge[lv].Update(int64(s.LevelDurations[lv]))
 }
 
-func (db *levelDB) GetProperty(name string) string {
-	return db.GetProperty(name)
-}
-
 func (db *levelDB) TryCatchUpWithPrimary() error {
 	return nil
 }

storage/database/memory_database.go

@@ -194,10 +194,6 @@ func (db *MemDB) Meter(prefix string) {
 	logger.Warn("MemDB does not support metrics!")
 }
 
-func (db *MemDB) GetProperty(name string) string {
-	return ""
-}
-
 func (db *MemDB) TryCatchUpWithPrimary() error {
 	return nil
 }

storage/database/rocksdb_database.go

@@ -216,10 +216,6 @@ func (db *rocksDB) Delete(key []byte) error {
 	return db.db.Delete(db.wo, key)
 }
 
-func (db *rocksDB) GetProperty(name string) string {
-	return db.db.GetProperty(name)
-}
-
 func (db *rocksDB) TryCatchUpWithPrimary() error {
 	return db.db.TryCatchUpWithPrimary()
 }