diff --git a/.changelog/unreleased/breaking-changes/111-compaction-support.md b/.changelog/unreleased/breaking-changes/111-compaction-support.md new file mode 100644 index 0000000..1551e0d --- /dev/null +++ b/.changelog/unreleased/breaking-changes/111-compaction-support.md @@ -0,0 +1 @@ +- Expanded db interface to support compaction ([\#111](https://github.com/cometbft/cometbft-db/pull/111)) diff --git a/.changelog/unreleased/features/111-compaction-support.md b/.changelog/unreleased/features/111-compaction-support.md new file mode 100644 index 0000000..05f38ec --- /dev/null +++ b/.changelog/unreleased/features/111-compaction-support.md @@ -0,0 +1 @@ +- Add compaction support to the databases ([\#111](https://github.com/cometbft/cometbft-db/pull/111)) diff --git a/backend_test.go b/backend_test.go index eeb7c0c..7bab0b8 100644 --- a/backend_test.go +++ b/backend_test.go @@ -4,7 +4,9 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -37,9 +39,10 @@ func testBackendGetSetDelete(t *testing.T, backend BackendType) { // Default dirname, err := os.MkdirTemp("", fmt.Sprintf("test_backend_%s_", backend)) require.Nil(t, err) - db, err := NewDB("testdb", backend, dirname) + name := fmt.Sprintf("testdb_%x", randStr(12)) + db, err := NewDB(name, backend, dirname) require.NoError(t, err) - defer cleanupDBDir(dirname, "testdb") + defer cleanupDBDir(dirname, name) // A nonexistent key should return nil. value, err := db.Get([]byte("a")) @@ -133,6 +136,26 @@ func testBackendGetSetDelete(t *testing.T, backend BackendType) { value, err = db.Get([]byte("x")) require.NoError(t, err) require.Equal(t, []byte{}, value) + + err = db.Compact(nil, nil) + if strings.Contains(string(backend), "pebbledb") { + // In pebble the start and end will be the same so + // we expect an error + require.Error(t, err) + } + + err = db.Set([]byte("y"), []byte{}) + require.NoError(t, err) + + err = db.Compact(nil, nil) + require.NoError(t, err) + + if strings.Contains(string(backend), "pebbledb") { + // When running the test the folder can't be cleaned up and there + // is a panic on removing the tmp testing directories. + // The compaction process is slow to release the lock on the folder. + time.Sleep(time.Second * 5) + } } func TestBackendsGetSetDelete(t *testing.T) { @@ -306,6 +329,7 @@ func testDBIterator(t *testing.T, backend BackendType) { // Ensure that the iterators don't panic with an empty database. dir2, err := os.MkdirTemp("", "tm-db-test") require.NoError(t, err) + name = fmt.Sprintf("test_%x", randStr(12)) db2, err := NewDB(name, backend, dir2) require.NoError(t, err) defer cleanupDBDir(dir2, name) diff --git a/badger_db.go b/badger_db.go index 77a8be2..d66f05c 100644 --- a/badger_db.go +++ b/badger_db.go @@ -170,6 +170,11 @@ func (b *BadgerDB) Stats() map[string]string { return nil } +func (b *BadgerDB) Compact(start, end []byte) error { + // Explicit compaction is not currently supported in badger + return nil +} + func (b *BadgerDB) NewBatch() Batch { wb := &badgerDBBatch{ db: b.db, diff --git a/boltdb.go b/boltdb.go index 1545be9..0df5296 100644 --- a/boltdb.go +++ b/boltdb.go @@ -203,3 +203,9 @@ func (bdb *BoltDB) ReverseIterator(start, end []byte) (Iterator, error) { } return newBoltDBIterator(tx, start, end, true), nil } + +func (bdb *BoltDB) Compact(start, end []byte) error { + // There is no explicit CompactRange support in BoltDB, only a function that copies the + // entire DB from one place to another while doing deletions. Hence we do not support it. + return nil +} diff --git a/cleveldb.go b/cleveldb.go index 3a42a31..2fb1df7 100644 --- a/cleveldb.go +++ b/cleveldb.go @@ -122,6 +122,13 @@ func (db *CLevelDB) DeleteSync(key []byte) error { return nil } +// Compact implements DB and compacts the given range of the DB +func (db *CLevelDB) Compact(start, end []byte) error { + // CompactRange of clevelDB does not return anything + db.db.CompactRange(levigo.Range{Start: start, Limit: end}) + return nil +} + // FIXME This should not be exposed func (db *CLevelDB) DB() *levigo.DB { return db.db diff --git a/goleveldb.go b/goleveldb.go index 518c729..e54f91b 100644 --- a/goleveldb.go +++ b/goleveldb.go @@ -33,6 +33,7 @@ func NewGoLevelDBWithOpts(name string, dir string, o *opt.Options) (*GoLevelDB, if err != nil { return nil, err } + database := &GoLevelDB{ db: db, } @@ -71,7 +72,8 @@ func (db *GoLevelDB) Set(key []byte, value []byte) error { if value == nil { return errValueNil } - if err := db.db.Put(key, value, nil); err != nil { + err := db.db.Put(key, value, nil) + if err != nil { return err } return nil @@ -85,7 +87,9 @@ func (db *GoLevelDB) SetSync(key []byte, value []byte) error { if value == nil { return errValueNil } - if err := db.db.Put(key, value, &opt.WriteOptions{Sync: true}); err != nil { + + err := db.db.Put(key, value, &opt.WriteOptions{Sync: true}) + if err != nil { return err } return nil @@ -96,7 +100,9 @@ func (db *GoLevelDB) Delete(key []byte) error { if len(key) == 0 { return errKeyEmpty } - if err := db.db.Delete(key, nil); err != nil { + + err := db.db.Delete(key, nil) + if err != nil { return err } return nil @@ -188,3 +194,8 @@ func (db *GoLevelDB) ReverseIterator(start, end []byte) (Iterator, error) { itr := db.db.NewIterator(&util.Range{Start: start, Limit: end}, nil) return newGoLevelDBIterator(itr, start, end, true), nil } + +// Compact range +func (db *GoLevelDB) Compact(start, end []byte) error { + return db.db.CompactRange(util.Range{Start: start, Limit: end}) +} diff --git a/goleveldb_batch.go b/goleveldb_batch.go index 4c1c6a2..4db2bd0 100644 --- a/goleveldb_batch.go +++ b/goleveldb_batch.go @@ -60,6 +60,7 @@ func (b *goLevelDBBatch) write(sync bool) error { if b.batch == nil { return errBatchClosed } + err := b.db.db.Write(b.batch, &opt.WriteOptions{Sync: sync}) if err != nil { return err diff --git a/memdb.go b/memdb.go index ba66c7b..b915504 100644 --- a/memdb.go +++ b/memdb.go @@ -208,3 +208,8 @@ func (db *MemDB) ReverseIteratorNoMtx(start, end []byte) (Iterator, error) { } return newMemDBIteratorMtxChoice(db, start, end, true, false), nil } + +func (*MemDB) Compact(start, end []byte) error { + // No Compaction is supported for memDB and there is no point in supporting compaction for a memory DB + return nil +} diff --git a/pebble.go b/pebble.go index 20f68a1..946c23b 100644 --- a/pebble.go +++ b/pebble.go @@ -136,6 +136,34 @@ func (db *PebbleDB) DB() *pebble.DB { return db.db } +func (db *PebbleDB) Compact(start, end []byte) (err error) { + // Currently nil,nil is an invalid range in Pebble. + // This was taken from https://github.com/cockroachdb/pebble/issues/1474 + // In case the start and end keys are the same + // pebbleDB will throw an error that it cannot compact. + if start != nil && end != nil { + return db.db.Compact(start, end, true) + } + iter, err := db.db.NewIter(nil) + if err != nil { + return err + } + defer func() { + err2 := iter.Close() + if err2 != nil { + err = err2 + } + }() + if start == nil && iter.First() { + start = append(start, iter.Key()...) + } + if end == nil && iter.Last() { + end = append(end, iter.Key()...) + } + err = db.db.Compact(start, end, true) + return +} + // Close implements DB. func (db PebbleDB) Close() error { db.db.Close() diff --git a/prefixdb.go b/prefixdb.go index 0b2d2a1..30e29a5 100644 --- a/prefixdb.go +++ b/prefixdb.go @@ -202,3 +202,7 @@ func (pdb *PrefixDB) Stats() map[string]string { func (pdb *PrefixDB) prefixed(key []byte) []byte { return append(cp(pdb.prefix), key...) } + +func (pdb *PrefixDB) Compact(start, end []byte) error { + return pdb.db.Compact(start, end) +} diff --git a/rocksdb.go b/rocksdb.go index d508b82..965fee0 100644 --- a/rocksdb.go +++ b/rocksdb.go @@ -202,3 +202,8 @@ func (db *RocksDB) ReverseIterator(start, end []byte) (Iterator, error) { itr := db.db.NewIterator(db.ro) return newRocksDBIterator(itr, start, end, true), nil } + +func (db *RocksDB) Compact(start, end []byte) error { + db.db.CompactRange(grocksdb.Range{Start: start, Limit: end}) + return nil +} diff --git a/types.go b/types.go index 131174d..74da21e 100644 --- a/types.go +++ b/types.go @@ -68,6 +68,9 @@ type DB interface { // Stats returns a map of property values for all keys and the size of the cache. Stats() map[string]string + + // Compact explicitly + Compact(start, end []byte) error } // Batch represents a group of writes. They may or may not be written atomically depending on the