added support for explicit compaction (#111)

* Added compaction to bolt and badger

* Added compaction to  pebble and removed stats from goleveldb

* Fixed lint

* Added compact function to clevelDB

* Added compact function to batch DBs

* Removed unused identifiers

* Added call to compact from tests and expanded the batch interface

* Added compacton to test

* Added compacton to test-fix

* Fixed tests for pebbleDB

* Fixed lint

* Added changelog

* Fixed typo

Co-authored-by: Adi Seredinschi <a@seredinschi.net>

* Use go 1.20

* Added comment on sleep in test

* Removed compact from batch

* Close iterator in defer

* Reverted downgrading go version

* Fixed compilation error

---------

Co-authored-by: Adi Seredinschi <a@seredinschi.net>

.changelog/unreleased/breaking-changes/111-compaction-support.md

@@ -0,0 +1 @@
+- Expanded db interface to support compaction ([\#111](https://github.com/cometbft/cometbft-db/pull/111))

.changelog/unreleased/features/111-compaction-support.md

@@ -0,0 +1 @@
+- Add compaction support to the databases ([\#111](https://github.com/cometbft/cometbft-db/pull/111))

backend_test.go

@@ -4,7 +4,9 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -37,9 +39,10 @@ func testBackendGetSetDelete(t *testing.T, backend BackendType) {
 	// Default
 	dirname, err := os.MkdirTemp("", fmt.Sprintf("test_backend_%s_", backend))
 	require.Nil(t, err)
-	db, err := NewDB("testdb", backend, dirname)
+	name := fmt.Sprintf("testdb_%x", randStr(12))
+	db, err := NewDB(name, backend, dirname)
 	require.NoError(t, err)
-	defer cleanupDBDir(dirname, "testdb")
+	defer cleanupDBDir(dirname, name)
 
 	// A nonexistent key should return nil.
 	value, err := db.Get([]byte("a"))
@@ -133,6 +136,26 @@ func testBackendGetSetDelete(t *testing.T, backend BackendType) {
 	value, err = db.Get([]byte("x"))
 	require.NoError(t, err)
 	require.Equal(t, []byte{}, value)
+
+	err = db.Compact(nil, nil)
+	if strings.Contains(string(backend), "pebbledb") {
+		// In pebble the start and end will be the same so
+		// we expect an error
+		require.Error(t, err)
+	}
+
+	err = db.Set([]byte("y"), []byte{})
+	require.NoError(t, err)
+
+	err = db.Compact(nil, nil)
+	require.NoError(t, err)
+
+	if strings.Contains(string(backend), "pebbledb") {
+		// When running the test the folder can't be cleaned up and there
+		// is a panic on removing the tmp testing directories.
+		// The compaction process is slow to release the lock on the folder.
+		time.Sleep(time.Second * 5)
+	}
 }
 
 func TestBackendsGetSetDelete(t *testing.T) {
@@ -306,6 +329,7 @@ func testDBIterator(t *testing.T, backend BackendType) {
 	// Ensure that the iterators don't panic with an empty database.
 	dir2, err := os.MkdirTemp("", "tm-db-test")
 	require.NoError(t, err)
+	name = fmt.Sprintf("test_%x", randStr(12))
 	db2, err := NewDB(name, backend, dir2)
 	require.NoError(t, err)
 	defer cleanupDBDir(dir2, name)

badger_db.go

@@ -170,6 +170,11 @@ func (b *BadgerDB) Stats() map[string]string {
 	return nil
 }
 
+func (b *BadgerDB) Compact(start, end []byte) error {
+	// Explicit compaction is not currently supported in badger
+	return nil
+}
+
 func (b *BadgerDB) NewBatch() Batch {
 	wb := &badgerDBBatch{
 		db:         b.db,

boltdb.go

@@ -203,3 +203,9 @@ func (bdb *BoltDB) ReverseIterator(start, end []byte) (Iterator, error) {
 	}
 	return newBoltDBIterator(tx, start, end, true), nil
 }
+
+func (bdb *BoltDB) Compact(start, end []byte) error {
+	// There is no explicit CompactRange support in BoltDB, only a function that copies the
+	// entire DB from one place to another while doing deletions. Hence we do not support it.
+	return nil
+}

cleveldb.go

@@ -122,6 +122,13 @@ func (db *CLevelDB) DeleteSync(key []byte) error {
 	return nil
 }
 
+// Compact implements DB and compacts the given range of the DB
+func (db *CLevelDB) Compact(start, end []byte) error {
+	// CompactRange of clevelDB does not return anything
+	db.db.CompactRange(levigo.Range{Start: start, Limit: end})
+	return nil
+}
+
 // FIXME This should not be exposed
 func (db *CLevelDB) DB() *levigo.DB {
 	return db.db

goleveldb.go

@@ -33,6 +33,7 @@ func NewGoLevelDBWithOpts(name string, dir string, o *opt.Options) (*GoLevelDB,
 	if err != nil {
 		return nil, err
 	}
+
 	database := &GoLevelDB{
 		db: db,
 	}
@@ -71,7 +72,8 @@ func (db *GoLevelDB) Set(key []byte, value []byte) error {
 	if value == nil {
 		return errValueNil
 	}
-	if err := db.db.Put(key, value, nil); err != nil {
+	err := db.db.Put(key, value, nil)
+	if err != nil {
 		return err
 	}
 	return nil
@@ -85,7 +87,9 @@ func (db *GoLevelDB) SetSync(key []byte, value []byte) error {
 	if value == nil {
 		return errValueNil
 	}
-	if err := db.db.Put(key, value, &opt.WriteOptions{Sync: true}); err != nil {
+
+	err := db.db.Put(key, value, &opt.WriteOptions{Sync: true})
+	if err != nil {
 		return err
 	}
 	return nil
@@ -96,7 +100,9 @@ func (db *GoLevelDB) Delete(key []byte) error {
 	if len(key) == 0 {
 		return errKeyEmpty
 	}
-	if err := db.db.Delete(key, nil); err != nil {
+
+	err := db.db.Delete(key, nil)
+	if err != nil {
 		return err
 	}
 	return nil
@@ -188,3 +194,8 @@ func (db *GoLevelDB) ReverseIterator(start, end []byte) (Iterator, error) {
 	itr := db.db.NewIterator(&util.Range{Start: start, Limit: end}, nil)
 	return newGoLevelDBIterator(itr, start, end, true), nil
 }
+
+// Compact range
+func (db *GoLevelDB) Compact(start, end []byte) error {
+	return db.db.CompactRange(util.Range{Start: start, Limit: end})
+}

goleveldb_batch.go

@@ -60,6 +60,7 @@ func (b *goLevelDBBatch) write(sync bool) error {
 	if b.batch == nil {
 		return errBatchClosed
 	}
+
 	err := b.db.db.Write(b.batch, &opt.WriteOptions{Sync: sync})
 	if err != nil {
 		return err

memdb.go

@@ -208,3 +208,8 @@ func (db *MemDB) ReverseIteratorNoMtx(start, end []byte) (Iterator, error) {
 	}
 	return newMemDBIteratorMtxChoice(db, start, end, true, false), nil
 }
+
+func (*MemDB) Compact(start, end []byte) error {
+	// No Compaction is supported for memDB and there is no point in supporting compaction for a memory DB
+	return nil
+}

pebble.go

@@ -136,6 +136,34 @@ func (db *PebbleDB) DB() *pebble.DB {
 	return db.db
 }
 
+func (db *PebbleDB) Compact(start, end []byte) (err error) {
+	// Currently nil,nil is an invalid range in Pebble.
+	// This was taken from https://github.com/cockroachdb/pebble/issues/1474
+	// In case the start and end keys are the same
+	// pebbleDB will throw an error that it cannot compact.
+	if start != nil && end != nil {
+		return db.db.Compact(start, end, true)
+	}
+	iter, err := db.db.NewIter(nil)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		err2 := iter.Close()
+		if err2 != nil {
+			err = err2
+		}
+	}()
+	if start == nil && iter.First() {
+		start = append(start, iter.Key()...)
+	}
+	if end == nil && iter.Last() {
+		end = append(end, iter.Key()...)
+	}
+	err = db.db.Compact(start, end, true)
+	return
+}
+
 // Close implements DB.
 func (db PebbleDB) Close() error {
 	db.db.Close()

prefixdb.go

@@ -202,3 +202,7 @@ func (pdb *PrefixDB) Stats() map[string]string {
 func (pdb *PrefixDB) prefixed(key []byte) []byte {
 	return append(cp(pdb.prefix), key...)
 }
+
+func (pdb *PrefixDB) Compact(start, end []byte) error {
+	return pdb.db.Compact(start, end)
+}

rocksdb.go

@@ -202,3 +202,8 @@ func (db *RocksDB) ReverseIterator(start, end []byte) (Iterator, error) {
 	itr := db.db.NewIterator(db.ro)
 	return newRocksDBIterator(itr, start, end, true), nil
 }
+
+func (db *RocksDB) Compact(start, end []byte) error {
+	db.db.CompactRange(grocksdb.Range{Start: start, Limit: end})
+	return nil
+}

types.go

@@ -68,6 +68,9 @@ type DB interface {
 
 	// Stats returns a map of property values for all keys and the size of the cache.
 	Stats() map[string]string
+
+	// Compact explicitly
+	Compact(start, end []byte) error
 }
 
 // Batch represents a group of writes. They may or may not be written atomically depending on the