diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1afa65b..ddfbb0f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,35 +9,18 @@ on: merge_group: pull_request: jobs: - cleanup-runs: - runs-on: ubuntu-latest - steps: - - uses: rokroskar/workflow-run-cleanup-action@master - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/main'" - Test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Set up Docker Build - uses: docker/setup-buildx-action@v3.2.0 - with: - driver: docker - - - name: Build Docker image - uses: docker/build-push-action@v5 + - uses: actions/setup-go@v4 with: - context: ./tools - file: ./tools/Dockerfile - tags: "cometbft/cometbft-db-testing:latest" - load: true + go-version: "1.22" + cache: false - name: test & coverage report creation - run: | - NON_INTERACTIVE=1 make docker-test + run: make test-all-with-coverage - uses: codecov/codecov-action@v4 with: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml deleted file mode 100644 index 7d4b180..0000000 --- a/.github/workflows/docker.yml +++ /dev/null @@ -1,64 +0,0 @@ -# This workflow builds and pushes a new version of the build container image -# when the tools directory changes on main. Edit tools/Dockerfile. -# -# This workflow does not push a new image until it is merged, so tests that -# depend on changes in this image will not pass until this workflow succeeds. -# For that reason, changes here should be done in a separate PR in advance of -# work that depends on them. -# -# To build the test image locally, run: -# make docker-test-image - -name: Docker testing image -on: - workflow_dispatch: - push: - branches: - - main - tags: - - "v[0-9]+.[0-9]+.[0-9]+" # Push events to matching v*, i.e. v1.0, v20.15.10 - - "v[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+" # e.g. v0.37.0-alpha.1, v0.38.0-alpha.10 - - "v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+" # e.g. v0.37.0-beta.1, v0.38.0-beta.10 - - "v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+" # e.g. v0.37.0-rc1, v0.38.0-rc10 - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Prepare - id: prep - run: | - DOCKER_IMAGE=cometbft/cometbft-db-testing - VERSION=noop - if [[ $GITHUB_REF == refs/tags/* ]]; then - VERSION=${GITHUB_REF#refs/tags/} - elif [[ $GITHUB_REF == refs/heads/* ]]; then - VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') - if [ "${{ github.event.repository.default_branch }}" = "$VERSION" ]; then - VERSION=latest - fi - fi - TAGS="${DOCKER_IMAGE}:${VERSION}" - if [[ $VERSION =~ ^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then - TAGS="$TAGS,${DOCKER_IMAGE}:${VERSION}" - fi - echo "tags=${TAGS}" >> $GITHUB_OUTPUT - - - name: Set up Docker Build - uses: docker/setup-buildx-action@v3.2.0 - - - name: Login to DockerHub - uses: docker/login-action@v3.1.0 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Publish to Docker Hub - uses: docker/build-push-action@v5 - with: - context: ./tools - file: ./tools/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.prep.outputs.tags }} diff --git a/.github/workflows/govulncheck.yml b/.github/workflows/govulncheck.yml index 2edad36..e0b18b0 100644 --- a/.github/workflows/govulncheck.yml +++ b/.github/workflows/govulncheck.yml @@ -18,16 +18,8 @@ jobs: steps: - uses: actions/setup-go@v5 with: - go-version: "1.21" + go-version: "1.22" check-latest: true - uses: actions/checkout@v4 - - uses: technote-space/get-diff-action@v6 - with: - PATTERNS: | - **/*.go - go.mod - go.sum - Makefile - name: govulncheck run: make vulncheck - if: "env.GIT_DIFF != ''" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 26d8e58..7672c5c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,21 +1,54 @@ -name: Lint +name: golangci-lint on: push: branches: - main pull_request: - merge_group: + +permissions: + contents: read + # Optional: allow read access to pull request. Use with `only-new-issues` option. + # pull-requests: read jobs: golangci: - # We need to run the linter on the same image we use for building, since it - # needs the C libraries installed for the dependencies to typecheck. + name: lint runs-on: ubuntu-latest - container: cometbft/cometbft-db-testing steps: - - uses: actions/checkout@v4 - - uses: golangci/golangci-lint-action@v4.0.0 + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: "1.22" + cache: false + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 with: - args: --timeout 10m - version: latest - github-token: ${{ secrets.github_token }} + # Require: The version of golangci-lint to use. + # When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version. + # When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit. + version: v1.56 + + # Optional: working directory, useful for monorepos + # working-directory: somedir + + # Optional: golangci-lint command line arguments. + # + # Note: By default, the `.golangci.yml` file should be at the root of the repository. + # The location of the configuration file can be changed by using `--config=` + # args: --timeout=30m --config=/my/path/.golangci.yml --issues-exit-code=0 + + # Optional: show only new issues if it's a pull request. The default value is `false`. + # only-new-issues: true + + # Optional: if set to true, then all caching functionality will be completely disabled, + # takes precedence over all other caching options. + # skip-cache: true + + # Optional: if set to true, then the action won't cache or restore ~/go/pkg. + # skip-pkg-cache: true + + # Optional: if set to true, then the action won't cache or restore ~/.cache/go-build. + # skip-build-cache: true + + # Optional: The mode to install golangci-lint. It can be 'binary' or 'goinstall'. + # install-mode: "goinstall" diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index b25cd59..06043b9 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -10,9 +10,7 @@ jobs: - uses: actions/stale@v9 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-pr-message: "This pull request has been automatically marked as stale because it has not had - recent activity. It will be closed if no further activity occurs. Thank you - for your contributions." + stale-pr-message: "This pull request has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions." days-before-stale: -1 days-before-close: -1 days-before-pr-stale: 10 diff --git a/Makefile b/Makefile index 9a14560..e374c8c 100644 --- a/Makefile +++ b/Makefile @@ -13,40 +13,11 @@ endif all: lint test ### go tests -## By default this will only test memdb & goleveldb test: @echo "--> Running go test" @go test $(PACKAGES) -v .PHONY: test -test-cleveldb: - @echo "--> Running go test" - @go test $(PACKAGES) -tags cleveldb -v -.PHONY: test-cleveldb - -test-rocksdb: - @echo "--> Running go test" - @go test $(PACKAGES) -tags rocksdb -v -.PHONY: test-rocksdb - -test-boltdb: - @echo "--> Running go test" - @go test $(PACKAGES) -tags boltdb -v -.PHONY: test-boltdb - -test-badgerdb: - @echo "--> Running go test" - @go test $(PACKAGES) -tags badgerdb -v -.PHONY: test-badgerdb - -test-pebble: - @echo "--> Running go test" - @go test $(PACKAGES) -tags pebbledb -v - -test-all: - @echo "--> Running go test" - @go test $(PACKAGES) -tags cleveldb,boltdb,rocksdb,grocksdb_clean_link,badgerdb,pebbledb -v -.PHONY: test-all test-all-with-coverage: @echo "--> Running go test for all databases, with coverage" @@ -56,7 +27,6 @@ test-all-with-coverage: -race \ -coverprofile=coverage.txt \ -covermode=atomic \ - -tags=memdb,goleveldb,cleveldb,boltdb,rocksdb,grocksdb_clean_link,badgerdb,pebbledb \ -v .PHONY: test-all-with-coverage @@ -77,16 +47,7 @@ docker-test-image: docker build -t $(DOCKER_TEST_IMAGE):$(DOCKER_TEST_IMAGE_VERSION) . .PHONY: docker-test-image -# Runs the same test as is executed in CI, but locally. -docker-test: - @echo "--> Running all tests with all databases with Docker (interactive flags: \"$(DOCKER_TEST_INTERACTIVE_FLAGS)\")" - @docker run $(DOCKER_TEST_INTERACTIVE_FLAGS) --rm --name cometbft-db-test \ - -v `pwd`:/cometbft \ - -w /cometbft \ - --entrypoint "" \ - $(DOCKER_TEST_IMAGE):$(DOCKER_TEST_IMAGE_VERSION) \ - make test-all-with-coverage -.PHONY: docker-test + tools: go get -v $(GOTOOLS) diff --git a/README.md b/README.md index fbd0cd7..2a55a2a 100644 --- a/README.md +++ b/README.md @@ -26,38 +26,6 @@ Go 1.21+ implementation of [LevelDB](https://github.com/google/leveldb) (see below). Currently the default on-disk database used in the Cosmos SDK. -- **MemDB [stable]:** An in-memory database using [Google's B-tree - package](https://github.com/google/btree). Has very high performance both for - reads, writes, and range scans, but is not durable and will lose all data on - process exit. Does not support transactions. Suitable for e.g. caches, working - sets, and tests. Used for [IAVL](https://github.com/tendermint/iavl) working - sets when the pruning strategy allows it. - -- **[LevelDB](https://github.com/google/leveldb) [experimental]:** A [Go - wrapper](https://github.com/jmhodges/levigo) around - [LevelDB](https://github.com/google/leveldb). Uses LSM-trees for on-disk - storage, which have good performance for write-heavy workloads, particularly - on spinning disks, but requires periodic compaction to maintain decent read - performance and reclaim disk space. Does not support transactions. - -- **[BoltDB](https://github.com/etcd-io/bbolt) [experimental]:** A - [fork](https://github.com/etcd-io/bbolt) of - [BoltDB](https://github.com/boltdb/bolt). Uses B+trees for on-disk storage, - which have good performance for read-heavy workloads and range scans. Supports - serializable ACID transactions. - -- **[RocksDB](https://github.com/linxGnu/grocksdb) [experimental]:** A [Go - wrapper](https://github.com/linxGnu/grocksdb) around - [RocksDB](https://rocksdb.org). Similarly to LevelDB (above) it uses LSM-trees - for on-disk storage, but is optimized for fast storage media such as SSDs and - memory. Supports atomic transactions, but not full ACID transactions. - -- **[BadgerDB](https://github.com/dgraph-io/badger) [experimental]:** A - key-value database written as a pure-Go alternative to e.g. LevelDB and - RocksDB, with LSM-tree storage. Makes use of multiple goroutines for - performance, and includes advanced features such as serializable ACID - transactions, write batches, compression, and more. - - **[PebbleDB](https://github.com/cockroachdb/pebble) [experimental]:** Pebble is a LevelDB/RocksDB inspired key-value store focused on performance and internal usage by CockroachDB. Pebble inherits the RocksDB file formats and a diff --git a/badger_db.go b/badger_db.go index d66f05c..9b412d5 100644 --- a/badger_db.go +++ b/badger_db.go @@ -1,6 +1,3 @@ -//go:build badgerdb -// +build badgerdb - package db import ( @@ -9,7 +6,7 @@ import ( "os" "path/filepath" - "github.com/dgraph-io/badger/v2" + "github.com/dgraph-io/badger/v4" ) func init() { registerDBCreator(BadgerDBBackend, badgerDBCreator) } @@ -128,7 +125,7 @@ func (b *BadgerDB) Close() error { return b.db.Close() } -func (b *BadgerDB) Print() error { +func (*BadgerDB) Print() error { return nil } @@ -166,11 +163,11 @@ func (b *BadgerDB) ReverseIterator(start, end []byte) (Iterator, error) { return b.iteratorOpts(end, start, opts) } -func (b *BadgerDB) Stats() map[string]string { +func (*BadgerDB) Stats() map[string]string { return nil } -func (b *BadgerDB) Compact(start, end []byte) error { +func (*BadgerDB) Compact(start, end []byte) error { // Explicit compaction is not currently supported in badger return nil } diff --git a/boltdb.go b/boltdb.go deleted file mode 100644 index 0df5296..0000000 --- a/boltdb.go +++ /dev/null @@ -1,211 +0,0 @@ -//go:build boltdb -// +build boltdb - -package db - -import ( - "errors" - "fmt" - "os" - "path/filepath" - - "go.etcd.io/bbolt" -) - -var bucket = []byte("tm") - -func init() { - registerDBCreator(BoltDBBackend, func(name, dir string) (DB, error) { - return NewBoltDB(name, dir) - }) -} - -// BoltDB is a wrapper around etcd's fork of bolt (https://github.com/etcd-io/bbolt). -// -// NOTE: All operations (including Set, Delete) are synchronous by default. One -// can globally turn it off by using NoSync config option (not recommended). -// -// A single bucket ([]byte("tm")) is used per a database instance. This could -// lead to performance issues when/if there will be lots of keys. -type BoltDB struct { - db *bbolt.DB -} - -var _ DB = (*BoltDB)(nil) - -// NewBoltDB returns a BoltDB with default options. -func NewBoltDB(name, dir string) (DB, error) { - return NewBoltDBWithOpts(name, dir, bbolt.DefaultOptions) -} - -// NewBoltDBWithOpts allows you to supply *bbolt.Options. ReadOnly: true is not -// supported because NewBoltDBWithOpts creates a global bucket. -func NewBoltDBWithOpts(name string, dir string, opts *bbolt.Options) (DB, error) { - if opts.ReadOnly { - return nil, errors.New("ReadOnly: true is not supported") - } - - dbPath := filepath.Join(dir, name+".db") - db, err := bbolt.Open(dbPath, os.ModePerm, opts) - if err != nil { - return nil, err - } - - // create a global bucket - err = db.Update(func(tx *bbolt.Tx) error { - _, err := tx.CreateBucketIfNotExists(bucket) - return err - }) - if err != nil { - return nil, err - } - - return &BoltDB{db: db}, nil -} - -// Get implements DB. -func (bdb *BoltDB) Get(key []byte) (value []byte, err error) { - if len(key) == 0 { - return nil, errKeyEmpty - } - err = bdb.db.View(func(tx *bbolt.Tx) error { - b := tx.Bucket(bucket) - if v := b.Get(key); v != nil { - value = append([]byte{}, v...) - } - return nil - }) - if err != nil { - return nil, err - } - return -} - -// Has implements DB. -func (bdb *BoltDB) Has(key []byte) (bool, error) { - bytes, err := bdb.Get(key) - if err != nil { - return false, err - } - return bytes != nil, nil -} - -// Set implements DB. -func (bdb *BoltDB) Set(key, value []byte) error { - if len(key) == 0 { - return errKeyEmpty - } - if value == nil { - return errValueNil - } - err := bdb.db.Update(func(tx *bbolt.Tx) error { - b := tx.Bucket(bucket) - return b.Put(key, value) - }) - if err != nil { - return err - } - return nil -} - -// SetSync implements DB. -func (bdb *BoltDB) SetSync(key, value []byte) error { - return bdb.Set(key, value) -} - -// Delete implements DB. -func (bdb *BoltDB) Delete(key []byte) error { - if len(key) == 0 { - return errKeyEmpty - } - err := bdb.db.Update(func(tx *bbolt.Tx) error { - return tx.Bucket(bucket).Delete(key) - }) - if err != nil { - return err - } - return nil -} - -// DeleteSync implements DB. -func (bdb *BoltDB) DeleteSync(key []byte) error { - return bdb.Delete(key) -} - -// Close implements DB. -func (bdb *BoltDB) Close() error { - return bdb.db.Close() -} - -// Print implements DB. -func (bdb *BoltDB) Print() error { - stats := bdb.db.Stats() - fmt.Printf("%v\n", stats) - - err := bdb.db.View(func(tx *bbolt.Tx) error { - tx.Bucket(bucket).ForEach(func(k, v []byte) error { - fmt.Printf("[%X]:\t[%X]\n", k, v) - return nil - }) - return nil - }) - if err != nil { - return err - } - return nil -} - -// Stats implements DB. -func (bdb *BoltDB) Stats() map[string]string { - stats := bdb.db.Stats() - m := make(map[string]string) - - // Freelist stats - m["FreePageN"] = fmt.Sprintf("%v", stats.FreePageN) - m["PendingPageN"] = fmt.Sprintf("%v", stats.PendingPageN) - m["FreeAlloc"] = fmt.Sprintf("%v", stats.FreeAlloc) - m["FreelistInuse"] = fmt.Sprintf("%v", stats.FreelistInuse) - - // Transaction stats - m["TxN"] = fmt.Sprintf("%v", stats.TxN) - m["OpenTxN"] = fmt.Sprintf("%v", stats.OpenTxN) - - return m -} - -// NewBatch implements DB. -func (bdb *BoltDB) NewBatch() Batch { - return newBoltDBBatch(bdb) -} - -// WARNING: Any concurrent writes or reads will block until the iterator is -// closed. -func (bdb *BoltDB) Iterator(start, end []byte) (Iterator, error) { - if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { - return nil, errKeyEmpty - } - tx, err := bdb.db.Begin(false) - if err != nil { - return nil, err - } - return newBoltDBIterator(tx, start, end, false), nil -} - -// WARNING: Any concurrent writes or reads will block until the iterator is -// closed. -func (bdb *BoltDB) ReverseIterator(start, end []byte) (Iterator, error) { - if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { - return nil, errKeyEmpty - } - tx, err := bdb.db.Begin(false) - if err != nil { - return nil, err - } - return newBoltDBIterator(tx, start, end, true), nil -} - -func (bdb *BoltDB) Compact(start, end []byte) error { - // There is no explicit CompactRange support in BoltDB, only a function that copies the - // entire DB from one place to another while doing deletions. Hence we do not support it. - return nil -} diff --git a/boltdb_batch.go b/boltdb_batch.go deleted file mode 100644 index cd22c67..0000000 --- a/boltdb_batch.go +++ /dev/null @@ -1,87 +0,0 @@ -//go:build boltdb -// +build boltdb - -package db - -import "go.etcd.io/bbolt" - -// boltDBBatch stores operations internally and dumps them to BoltDB on Write(). -type boltDBBatch struct { - db *BoltDB - ops []operation -} - -var _ Batch = (*boltDBBatch)(nil) - -func newBoltDBBatch(db *BoltDB) *boltDBBatch { - return &boltDBBatch{ - db: db, - ops: []operation{}, - } -} - -// Set implements Batch. -func (b *boltDBBatch) Set(key, value []byte) error { - if len(key) == 0 { - return errKeyEmpty - } - if value == nil { - return errValueNil - } - if b.ops == nil { - return errBatchClosed - } - b.ops = append(b.ops, operation{opTypeSet, key, value}) - return nil -} - -// Delete implements Batch. -func (b *boltDBBatch) Delete(key []byte) error { - if len(key) == 0 { - return errKeyEmpty - } - if b.ops == nil { - return errBatchClosed - } - b.ops = append(b.ops, operation{opTypeDelete, key, nil}) - return nil -} - -// Write implements Batch. -func (b *boltDBBatch) Write() error { - if b.ops == nil { - return errBatchClosed - } - err := b.db.db.Batch(func(tx *bbolt.Tx) error { - bkt := tx.Bucket(bucket) - for _, op := range b.ops { - switch op.opType { - case opTypeSet: - if err := bkt.Put(op.key, op.value); err != nil { - return err - } - case opTypeDelete: - if err := bkt.Delete(op.key); err != nil { - return err - } - } - } - return nil - }) - if err != nil { - return err - } - // Make sure batch cannot be used afterwards. Callers should still call Close(), for errors. - return b.Close() -} - -// WriteSync implements Batch. -func (b *boltDBBatch) WriteSync() error { - return b.Write() -} - -// Close implements Batch. -func (b *boltDBBatch) Close() error { - b.ops = nil - return nil -} diff --git a/boltdb_iterator.go b/boltdb_iterator.go deleted file mode 100644 index a62e2ab..0000000 --- a/boltdb_iterator.go +++ /dev/null @@ -1,142 +0,0 @@ -//go:build boltdb -// +build boltdb - -package db - -import ( - "bytes" - - "go.etcd.io/bbolt" -) - -// boltDBIterator allows you to iterate on range of keys/values given some -// start / end keys (nil & nil will result in doing full scan). -type boltDBIterator struct { - tx *bbolt.Tx - - itr *bbolt.Cursor - start []byte - end []byte - - currentKey []byte - currentValue []byte - - isInvalid bool - isReverse bool -} - -var _ Iterator = (*boltDBIterator)(nil) - -// newBoltDBIterator creates a new boltDBIterator. -func newBoltDBIterator(tx *bbolt.Tx, start, end []byte, isReverse bool) *boltDBIterator { - itr := tx.Bucket(bucket).Cursor() - - var ck, cv []byte - if isReverse { - switch { - case end == nil: - ck, cv = itr.Last() - default: - _, _ = itr.Seek(end) // after key - ck, cv = itr.Prev() // return to end key - } - } else { - switch { - case start == nil: - ck, cv = itr.First() - default: - ck, cv = itr.Seek(start) - } - } - - return &boltDBIterator{ - tx: tx, - itr: itr, - start: start, - end: end, - currentKey: ck, - currentValue: cv, - isReverse: isReverse, - isInvalid: false, - } -} - -// Domain implements Iterator. -func (itr *boltDBIterator) Domain() ([]byte, []byte) { - return itr.start, itr.end -} - -// Valid implements Iterator. -func (itr *boltDBIterator) Valid() bool { - if itr.isInvalid { - return false - } - - if itr.Error() != nil { - itr.isInvalid = true - return false - } - - // iterated to the end of the cursor - if itr.currentKey == nil { - itr.isInvalid = true - return false - } - - if itr.isReverse { - if itr.start != nil && bytes.Compare(itr.currentKey, itr.start) < 0 { - itr.isInvalid = true - return false - } - } else { - if itr.end != nil && bytes.Compare(itr.end, itr.currentKey) <= 0 { - itr.isInvalid = true - return false - } - } - - // Valid - return true -} - -// Next implements Iterator. -func (itr *boltDBIterator) Next() { - itr.assertIsValid() - if itr.isReverse { - itr.currentKey, itr.currentValue = itr.itr.Prev() - } else { - itr.currentKey, itr.currentValue = itr.itr.Next() - } -} - -// Key implements Iterator. -func (itr *boltDBIterator) Key() []byte { - itr.assertIsValid() - return append([]byte{}, itr.currentKey...) -} - -// Value implements Iterator. -func (itr *boltDBIterator) Value() []byte { - itr.assertIsValid() - var value []byte - if itr.currentValue != nil { - value = append([]byte{}, itr.currentValue...) - } - return value -} - -// Error implements Iterator. -func (itr *boltDBIterator) Error() error { - return nil -} - -// Close implements Iterator. -func (itr *boltDBIterator) Close() error { - return itr.tx.Rollback() -} - -func (itr *boltDBIterator) assertIsValid() { - if !itr.Valid() { - panic("iterator is invalid") - } -} diff --git a/boltdb_test.go b/boltdb_test.go deleted file mode 100644 index e68c85b..0000000 --- a/boltdb_test.go +++ /dev/null @@ -1,36 +0,0 @@ -//go:build boltdb -// +build boltdb - -package db - -import ( - "fmt" - "os" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestBoltDBNewBoltDB(t *testing.T) { - name := fmt.Sprintf("test_%x", randStr(12)) - dir := os.TempDir() - defer cleanupDBDir(dir, name) - - db, err := NewBoltDB(name, dir) - require.NoError(t, err) - db.Close() -} - -func BenchmarkBoltDBRandomReadsWrites(b *testing.B) { - name := fmt.Sprintf("test_%x", randStr(12)) - db, err := NewBoltDB(name, "") - if err != nil { - b.Fatal(err) - } - defer func() { - db.Close() - cleanupDBDir("", name) - }() - - benchmarkRandomReadsWrites(b, db) -} diff --git a/cleveldb.go b/cleveldb.go deleted file mode 100644 index 2fb1df7..0000000 --- a/cleveldb.go +++ /dev/null @@ -1,203 +0,0 @@ -//go:build cleveldb -// +build cleveldb - -package db - -import ( - "fmt" - "path/filepath" - - "github.com/jmhodges/levigo" -) - -func init() { - dbCreator := func(name string, dir string) (DB, error) { - return NewCLevelDB(name, dir) - } - registerDBCreator(CLevelDBBackend, dbCreator) -} - -// CLevelDB uses the C LevelDB database via a Go wrapper. -type CLevelDB struct { - db *levigo.DB - ro *levigo.ReadOptions - wo *levigo.WriteOptions - woSync *levigo.WriteOptions -} - -var _ DB = (*CLevelDB)(nil) - -// NewCLevelDB creates a new CLevelDB. -func NewCLevelDB(name string, dir string) (*CLevelDB, error) { - dbPath := filepath.Join(dir, name+".db") - - opts := levigo.NewOptions() - opts.SetCache(levigo.NewLRUCache(1 << 30)) - opts.SetCreateIfMissing(true) - db, err := levigo.Open(dbPath, opts) - if err != nil { - return nil, err - } - ro := levigo.NewReadOptions() - wo := levigo.NewWriteOptions() - woSync := levigo.NewWriteOptions() - woSync.SetSync(true) - database := &CLevelDB{ - db: db, - ro: ro, - wo: wo, - woSync: woSync, - } - return database, nil -} - -// Get implements DB. -func (db *CLevelDB) Get(key []byte) ([]byte, error) { - if len(key) == 0 { - return nil, errKeyEmpty - } - res, err := db.db.Get(db.ro, key) - if err != nil { - return nil, err - } - return res, nil -} - -// Has implements DB. -func (db *CLevelDB) Has(key []byte) (bool, error) { - bytes, err := db.Get(key) - if err != nil { - return false, err - } - return bytes != nil, nil -} - -// Set implements DB. -func (db *CLevelDB) Set(key []byte, value []byte) error { - if len(key) == 0 { - return errKeyEmpty - } - if value == nil { - return errValueNil - } - if err := db.db.Put(db.wo, key, value); err != nil { - return err - } - return nil -} - -// SetSync implements DB. -func (db *CLevelDB) SetSync(key []byte, value []byte) error { - if len(key) == 0 { - return errKeyEmpty - } - if value == nil { - return errValueNil - } - if err := db.db.Put(db.woSync, key, value); err != nil { - return err - } - return nil -} - -// Delete implements DB. -func (db *CLevelDB) Delete(key []byte) error { - if len(key) == 0 { - return errKeyEmpty - } - if err := db.db.Delete(db.wo, key); err != nil { - return err - } - return nil -} - -// DeleteSync implements DB. -func (db *CLevelDB) DeleteSync(key []byte) error { - if len(key) == 0 { - return errKeyEmpty - } - if err := db.db.Delete(db.woSync, key); err != nil { - return err - } - return nil -} - -// Compact implements DB and compacts the given range of the DB -func (db *CLevelDB) Compact(start, end []byte) error { - // CompactRange of clevelDB does not return anything - db.db.CompactRange(levigo.Range{Start: start, Limit: end}) - return nil -} - -// FIXME This should not be exposed -func (db *CLevelDB) DB() *levigo.DB { - return db.db -} - -// Close implements DB. -func (db *CLevelDB) Close() error { - db.db.Close() - db.ro.Close() - db.wo.Close() - db.woSync.Close() - return nil -} - -// Print implements DB. -func (db *CLevelDB) Print() error { - itr, err := db.Iterator(nil, nil) - if err != nil { - return err - } - defer itr.Close() - for ; itr.Valid(); itr.Next() { - key := itr.Key() - value := itr.Value() - fmt.Printf("[%X]:\t[%X]\n", key, value) - } - return nil -} - -// Stats implements DB. -func (db *CLevelDB) Stats() map[string]string { - keys := []string{ - "leveldb.aliveiters", - "leveldb.alivesnaps", - "leveldb.blockpool", - "leveldb.cachedblock", - "leveldb.num-files-at-level{n}", - "leveldb.openedtables", - "leveldb.sstables", - "leveldb.stats", - } - - stats := make(map[string]string, len(keys)) - for _, key := range keys { - str := db.db.PropertyValue(key) - stats[key] = str - } - return stats -} - -// NewBatch implements DB. -func (db *CLevelDB) NewBatch() Batch { - return newCLevelDBBatch(db) -} - -// Iterator implements DB. -func (db *CLevelDB) Iterator(start, end []byte) (Iterator, error) { - if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { - return nil, errKeyEmpty - } - itr := db.db.NewIterator(db.ro) - return newCLevelDBIterator(itr, start, end, false), nil -} - -// ReverseIterator implements DB. -func (db *CLevelDB) ReverseIterator(start, end []byte) (Iterator, error) { - if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { - return nil, errKeyEmpty - } - itr := db.db.NewIterator(db.ro) - return newCLevelDBIterator(itr, start, end, true), nil -} diff --git a/cleveldb_batch.go b/cleveldb_batch.go deleted file mode 100644 index b77bd52..0000000 --- a/cleveldb_batch.go +++ /dev/null @@ -1,82 +0,0 @@ -//go:build cleveldb -// +build cleveldb - -package db - -import "github.com/jmhodges/levigo" - -// cLevelDBBatch is a LevelDB batch. -type cLevelDBBatch struct { - db *CLevelDB - batch *levigo.WriteBatch -} - -func newCLevelDBBatch(db *CLevelDB) *cLevelDBBatch { - return &cLevelDBBatch{ - db: db, - batch: levigo.NewWriteBatch(), - } -} - -// Set implements Batch. -func (b *cLevelDBBatch) Set(key, value []byte) error { - if len(key) == 0 { - return errKeyEmpty - } - if value == nil { - return errValueNil - } - if b.batch == nil { - return errBatchClosed - } - b.batch.Put(key, value) - return nil -} - -// Delete implements Batch. -func (b *cLevelDBBatch) Delete(key []byte) error { - if len(key) == 0 { - return errKeyEmpty - } - if b.batch == nil { - return errBatchClosed - } - b.batch.Delete(key) - return nil -} - -// Write implements Batch. -func (b *cLevelDBBatch) Write() error { - if b.batch == nil { - return errBatchClosed - } - err := b.db.db.Write(b.db.wo, b.batch) - if err != nil { - return err - } - // Make sure batch cannot be used afterwards. Callers should still call Close(), for errors. - return b.Close() -} - -// WriteSync implements Batch. -func (b *cLevelDBBatch) WriteSync() error { - if b.batch == nil { - return errBatchClosed - } - err := b.db.db.Write(b.db.woSync, b.batch) - if err != nil { - return err - } - // Make sure batch cannot be used afterwards. Callers should still call Close(), for errors. - b.Close() - return nil -} - -// Close implements Batch. -func (b *cLevelDBBatch) Close() error { - if b.batch != nil { - b.batch.Close() - b.batch = nil - } - return nil -} diff --git a/cleveldb_iterator.go b/cleveldb_iterator.go deleted file mode 100644 index cf5ac19..0000000 --- a/cleveldb_iterator.go +++ /dev/null @@ -1,134 +0,0 @@ -//go:build cleveldb -// +build cleveldb - -package db - -import ( - "bytes" - - "github.com/jmhodges/levigo" -) - -// cLevelDBIterator is a cLevelDB iterator. -type cLevelDBIterator struct { - source *levigo.Iterator - start, end []byte - isReverse bool - isInvalid bool -} - -var _ Iterator = (*cLevelDBIterator)(nil) - -func newCLevelDBIterator(source *levigo.Iterator, start, end []byte, isReverse bool) *cLevelDBIterator { - if isReverse { - if end == nil || len(end) == 0 { - source.SeekToLast() - } else { - source.Seek(end) - if source.Valid() { - eoakey := source.Key() // end or after key - if bytes.Compare(end, eoakey) <= 0 { - source.Prev() - } - } else { - source.SeekToLast() - } - } - } else { - if start == nil || len(start) == 0 { - source.SeekToFirst() - } else { - source.Seek(start) - } - } - return &cLevelDBIterator{ - source: source, - start: start, - end: end, - isReverse: isReverse, - isInvalid: false, - } -} - -// Domain implements Iterator. -func (itr cLevelDBIterator) Domain() ([]byte, []byte) { - return itr.start, itr.end -} - -// Valid implements Iterator. -func (itr cLevelDBIterator) Valid() bool { - // Once invalid, forever invalid. - if itr.isInvalid { - return false - } - - // If source errors, invalid. - if itr.source.GetError() != nil { - itr.isInvalid = true - return false - } - - // If source is invalid, invalid. - if !itr.source.Valid() { - itr.isInvalid = true - return false - } - - // If key is end or past it, invalid. - start := itr.start - end := itr.end - key := itr.source.Key() - if itr.isReverse { - if start != nil && bytes.Compare(key, start) < 0 { - itr.isInvalid = true - return false - } - } else { - if end != nil && bytes.Compare(end, key) <= 0 { - itr.isInvalid = true - return false - } - } - - // It's valid. - return true -} - -// Key implements Iterator. -func (itr cLevelDBIterator) Key() []byte { - itr.assertIsValid() - return itr.source.Key() -} - -// Value implements Iterator. -func (itr cLevelDBIterator) Value() []byte { - itr.assertIsValid() - return itr.source.Value() -} - -// Next implements Iterator. -func (itr cLevelDBIterator) Next() { - itr.assertIsValid() - if itr.isReverse { - itr.source.Prev() - } else { - itr.source.Next() - } -} - -// Error implements Iterator. -func (itr cLevelDBIterator) Error() error { - return itr.source.GetError() -} - -// Close implements Iterator. -func (itr cLevelDBIterator) Close() error { - itr.source.Close() - return nil -} - -func (itr cLevelDBIterator) assertIsValid() { - if !itr.Valid() { - panic("iterator is invalid") - } -} diff --git a/cleveldb_test.go b/cleveldb_test.go deleted file mode 100644 index 42a1bd9..0000000 --- a/cleveldb_test.go +++ /dev/null @@ -1,101 +0,0 @@ -//go:build cleveldb -// +build cleveldb - -package db - -import ( - "bytes" - "fmt" - "math/rand" - "os" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func BenchmarkRandomReadsWrites2(b *testing.B) { - b.StopTimer() - - numItems := int64(1000000) - internal := map[int64]int64{} - for i := 0; i < int(numItems); i++ { - internal[int64(i)] = int64(0) - } - db, err := NewCLevelDB(fmt.Sprintf("test_%x", randStr(12)), "") - if err != nil { - b.Fatal(err.Error()) - return - } - - b.StartTimer() - - for i := 0; i < b.N; i++ { - // Write something - { - idx := (int64(rand.Int()) % numItems) - internal[idx]++ - val := internal[idx] - idxBytes := int642Bytes(int64(idx)) - valBytes := int642Bytes(int64(val)) - db.Set( - idxBytes, - valBytes, - ) - } - // Read something - { - idx := (int64(rand.Int()) % numItems) - val := internal[idx] - idxBytes := int642Bytes(int64(idx)) - valBytes, err := db.Get(idxBytes) - if err != nil { - b.Error(err) - } - if val == 0 { - if !bytes.Equal(valBytes, nil) { - b.Errorf("Expected %v for %v, got %X", - nil, idx, valBytes) - break - } - } else { - if len(valBytes) != 8 { - b.Errorf("Expected length 8 for %v, got %X", - idx, valBytes) - break - } - valGot := bytes2Int64(valBytes) - if val != valGot { - b.Errorf("Expected %v for %v, got %v", - val, idx, valGot) - break - } - } - } - } - - db.Close() -} - -func TestCLevelDBBackend(t *testing.T) { - name := fmt.Sprintf("test_%x", randStr(12)) - // Can't use "" (current directory) or "./" here because levigo.Open returns: - // "Error initializing DB: IO error: test_XXX.db: Invalid argument" - dir := os.TempDir() - db, err := NewDB(name, CLevelDBBackend, dir) - require.NoError(t, err) - defer cleanupDBDir(dir, name) - - _, ok := db.(*CLevelDB) - assert.True(t, ok) -} - -func TestCLevelDBStats(t *testing.T) { - name := fmt.Sprintf("test_%x", randStr(12)) - dir := os.TempDir() - db, err := NewDB(name, CLevelDBBackend, dir) - require.NoError(t, err) - defer cleanupDBDir(dir, name) - - assert.NotEmpty(t, db.Stats()) -} diff --git a/go.mod b/go.mod index 8c8bff1..0e74602 100644 --- a/go.mod +++ b/go.mod @@ -1,35 +1,33 @@ module github.com/cometbft/cometbft-db -go 1.21 +go 1.22.1 require ( github.com/cockroachdb/pebble v1.1.0 - github.com/dgraph-io/badger/v2 v2.2007.4 + github.com/dgraph-io/badger/v4 v4.2.0 github.com/google/btree v1.1.2 - github.com/jmhodges/levigo v1.0.0 - github.com/linxGnu/grocksdb v1.8.12 github.com/stretchr/testify v1.9.0 github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca - go.etcd.io/bbolt v1.3.9 ) require ( github.com/DataDog/zstd v1.4.5 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cockroachdb/errors v1.11.1 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de // indirect - github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/getsentry/sentry-go v0.18.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/glog v1.0.0 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/flatbuffers v1.12.1 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/klauspost/compress v1.15.15 // indirect github.com/kr/pretty v0.3.1 // indirect @@ -42,8 +40,10 @@ require ( github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect + go.opencensus.io v0.22.5 // indirect golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df // indirect golang.org/x/net v0.18.0 // indirect + golang.org/x/sync v0.5.0 // indirect golang.org/x/sys v0.14.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/protobuf v1.31.0 // indirect diff --git a/go.sum b/go.sum index 8fce8f0..1166e92 100644 --- a/go.sum +++ b/go.sum @@ -35,21 +35,16 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= @@ -71,18 +66,14 @@ github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwP github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= -github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= -github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= -github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= +github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= @@ -112,8 +103,11 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -140,13 +134,14 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -173,12 +168,8 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U= -github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -190,7 +181,6 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -203,14 +193,9 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/linxGnu/grocksdb v1.8.12 h1:1/pCztQUOa3BX/1gR3jSZDoaKFpeHFvQ1XrqZpSvZVo= -github.com/linxGnu/grocksdb v1.8.12/go.mod h1:xZCIb5Muw+nhbDK4Y5UJuOrin5MceOuiXkVUR7vp4WY= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -227,7 +212,6 @@ github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9k github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -263,19 +247,9 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= -github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -285,21 +259,18 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca h1:Ld/zXl5t4+D69SiV4JoN7kkfvJdOWlPpfxrzxpLMoUk= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.etcd.io/bbolt v1.3.9 h1:8x7aARPEXiXbHmtUwAIv7eV2fQFHrLLavdiJ3uzJXoI= -go.etcd.io/bbolt v1.3.9/go.mod h1:zaO32+Ti0PK1ivdPtgMESzuzL2VPoIG1PCQNvOdo/dE= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -394,7 +365,6 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -403,7 +373,6 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -435,6 +404,7 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/pebble.go b/pebble.go index 946c23b..5249e73 100644 --- a/pebble.go +++ b/pebble.go @@ -1,6 +1,3 @@ -//go:build pebbledb -// +build pebbledb - package db import ( @@ -161,7 +158,7 @@ func (db *PebbleDB) Compact(start, end []byte) (err error) { end = append(end, iter.Key()...) } err = db.db.Compact(start, end, true) - return + return err } // Close implements DB. diff --git a/rocksdb.go b/rocksdb.go deleted file mode 100644 index 965fee0..0000000 --- a/rocksdb.go +++ /dev/null @@ -1,209 +0,0 @@ -//go:build rocksdb -// +build rocksdb - -package db - -import ( - "fmt" - "path/filepath" - "runtime" - - "github.com/linxGnu/grocksdb" -) - -func init() { - dbCreator := func(name string, dir string) (DB, error) { - return NewRocksDB(name, dir) - } - registerDBCreator(RocksDBBackend, dbCreator) -} - -// RocksDB is a RocksDB backend. -type RocksDB struct { - db *grocksdb.DB - ro *grocksdb.ReadOptions - wo *grocksdb.WriteOptions - woSync *grocksdb.WriteOptions -} - -var _ DB = (*RocksDB)(nil) - -func NewRocksDB(name string, dir string) (*RocksDB, error) { - // default rocksdb option, good enough for most cases, including heavy workloads. - // 1GB table cache, 512MB write buffer(may use 50% more on heavy workloads). - // compression: snappy as default, need to -lsnappy to enable. - bbto := grocksdb.NewDefaultBlockBasedTableOptions() - bbto.SetBlockCache(grocksdb.NewLRUCache(1 << 30)) - bbto.SetFilterPolicy(grocksdb.NewBloomFilter(10)) - - opts := grocksdb.NewDefaultOptions() - opts.SetBlockBasedTableFactory(bbto) - opts.SetCreateIfMissing(true) - opts.IncreaseParallelism(runtime.NumCPU()) - // 1.5GB maximum memory use for writebuffer. - opts.OptimizeLevelStyleCompaction(512 * 1024 * 1024) - return NewRocksDBWithOptions(name, dir, opts) -} - -func NewRocksDBWithOptions(name string, dir string, opts *grocksdb.Options) (*RocksDB, error) { - dbPath := filepath.Join(dir, name+".db") - db, err := grocksdb.OpenDb(opts, dbPath) - if err != nil { - return nil, err - } - ro := grocksdb.NewDefaultReadOptions() - wo := grocksdb.NewDefaultWriteOptions() - woSync := grocksdb.NewDefaultWriteOptions() - woSync.SetSync(true) - return NewRocksDBWithRawDB(db, ro, wo, woSync), nil -} - -func NewRocksDBWithRawDB(db *grocksdb.DB, ro *grocksdb.ReadOptions, wo *grocksdb.WriteOptions, woSync *grocksdb.WriteOptions) *RocksDB { - return &RocksDB{ - db: db, - ro: ro, - wo: wo, - woSync: woSync, - } -} - -// Get implements DB. -func (db *RocksDB) Get(key []byte) ([]byte, error) { - if len(key) == 0 { - return nil, errKeyEmpty - } - res, err := db.db.Get(db.ro, key) - if err != nil { - return nil, err - } - return moveSliceToBytes(res), nil -} - -// Has implements DB. -func (db *RocksDB) Has(key []byte) (bool, error) { - bytes, err := db.Get(key) - if err != nil { - return false, err - } - return bytes != nil, nil -} - -// Set implements DB. -func (db *RocksDB) Set(key []byte, value []byte) error { - if len(key) == 0 { - return errKeyEmpty - } - if value == nil { - return errValueNil - } - err := db.db.Put(db.wo, key, value) - if err != nil { - return err - } - return nil -} - -// SetSync implements DB. -func (db *RocksDB) SetSync(key []byte, value []byte) error { - if len(key) == 0 { - return errKeyEmpty - } - if value == nil { - return errValueNil - } - err := db.db.Put(db.woSync, key, value) - if err != nil { - return err - } - return nil -} - -// Delete implements DB. -func (db *RocksDB) Delete(key []byte) error { - if len(key) == 0 { - return errKeyEmpty - } - err := db.db.Delete(db.wo, key) - if err != nil { - return err - } - return nil -} - -// DeleteSync implements DB. -func (db *RocksDB) DeleteSync(key []byte) error { - if len(key) == 0 { - return errKeyEmpty - } - err := db.db.Delete(db.woSync, key) - if err != nil { - return nil - } - return nil -} - -func (db *RocksDB) DB() *grocksdb.DB { - return db.db -} - -// Close implements DB. -func (db *RocksDB) Close() error { - db.ro.Destroy() - db.wo.Destroy() - db.woSync.Destroy() - db.db.Close() - return nil -} - -// Print implements DB. -func (db *RocksDB) Print() error { - itr, err := db.Iterator(nil, nil) - if err != nil { - return err - } - defer itr.Close() - for ; itr.Valid(); itr.Next() { - key := itr.Key() - value := itr.Value() - fmt.Printf("[%X]:\t[%X]\n", key, value) - } - return nil -} - -// Stats implements DB. -func (db *RocksDB) Stats() map[string]string { - keys := []string{"rocksdb.stats"} - stats := make(map[string]string, len(keys)) - for _, key := range keys { - stats[key] = db.db.GetProperty(key) - } - return stats -} - -// NewBatch implements DB. -func (db *RocksDB) NewBatch() Batch { - return newRocksDBBatch(db) -} - -// Iterator implements DB. -func (db *RocksDB) Iterator(start, end []byte) (Iterator, error) { - if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { - return nil, errKeyEmpty - } - itr := db.db.NewIterator(db.ro) - return newRocksDBIterator(itr, start, end, false), nil -} - -// ReverseIterator implements DB. -func (db *RocksDB) ReverseIterator(start, end []byte) (Iterator, error) { - if (start != nil && len(start) == 0) || (end != nil && len(end) == 0) { - return nil, errKeyEmpty - } - itr := db.db.NewIterator(db.ro) - return newRocksDBIterator(itr, start, end, true), nil -} - -func (db *RocksDB) Compact(start, end []byte) error { - db.db.CompactRange(grocksdb.Range{Start: start, Limit: end}) - return nil -} diff --git a/rocksdb_batch.go b/rocksdb_batch.go deleted file mode 100644 index 6ebc8da..0000000 --- a/rocksdb_batch.go +++ /dev/null @@ -1,83 +0,0 @@ -//go:build rocksdb -// +build rocksdb - -package db - -import "github.com/linxGnu/grocksdb" - -type rocksDBBatch struct { - db *RocksDB - batch *grocksdb.WriteBatch -} - -var _ Batch = (*rocksDBBatch)(nil) - -func newRocksDBBatch(db *RocksDB) *rocksDBBatch { - return &rocksDBBatch{ - db: db, - batch: grocksdb.NewWriteBatch(), - } -} - -// Set implements Batch. -func (b *rocksDBBatch) Set(key, value []byte) error { - if len(key) == 0 { - return errKeyEmpty - } - if value == nil { - return errValueNil - } - if b.batch == nil { - return errBatchClosed - } - b.batch.Put(key, value) - return nil -} - -// Delete implements Batch. -func (b *rocksDBBatch) Delete(key []byte) error { - if len(key) == 0 { - return errKeyEmpty - } - if b.batch == nil { - return errBatchClosed - } - b.batch.Delete(key) - return nil -} - -// Write implements Batch. -func (b *rocksDBBatch) Write() error { - if b.batch == nil { - return errBatchClosed - } - err := b.db.db.Write(b.db.wo, b.batch) - if err != nil { - return err - } - // Make sure batch cannot be used afterwards. Callers should still call Close(), for errors. - b.Close() - return nil -} - -// WriteSync implements Batch. -func (b *rocksDBBatch) WriteSync() error { - if b.batch == nil { - return errBatchClosed - } - err := b.db.db.Write(b.db.woSync, b.batch) - if err != nil { - return err - } - // Make sure batch cannot be used afterwards. Callers should still call Close(), for errors. - return b.Close() -} - -// Close implements Batch. -func (b *rocksDBBatch) Close() error { - if b.batch != nil { - b.batch.Destroy() - b.batch = nil - } - return nil -} diff --git a/rocksdb_iterator.go b/rocksdb_iterator.go deleted file mode 100644 index 53fa9a9..0000000 --- a/rocksdb_iterator.go +++ /dev/null @@ -1,146 +0,0 @@ -//go:build rocksdb -// +build rocksdb - -package db - -import ( - "bytes" - - "github.com/linxGnu/grocksdb" -) - -type rocksDBIterator struct { - source *grocksdb.Iterator - start, end []byte - isReverse bool - isInvalid bool -} - -var _ Iterator = (*rocksDBIterator)(nil) - -func newRocksDBIterator(source *grocksdb.Iterator, start, end []byte, isReverse bool) *rocksDBIterator { - if isReverse { - if end == nil { - source.SeekToLast() - } else { - source.Seek(end) - if source.Valid() { - eoakey := moveSliceToBytes(source.Key()) // end or after key - if bytes.Compare(end, eoakey) <= 0 { - source.Prev() - } - } else { - source.SeekToLast() - } - } - } else { - if start == nil { - source.SeekToFirst() - } else { - source.Seek(start) - } - } - return &rocksDBIterator{ - source: source, - start: start, - end: end, - isReverse: isReverse, - isInvalid: false, - } -} - -// Domain implements Iterator. -func (itr *rocksDBIterator) Domain() ([]byte, []byte) { - return itr.start, itr.end -} - -// Valid implements Iterator. -func (itr *rocksDBIterator) Valid() bool { - // Once invalid, forever invalid. - if itr.isInvalid { - return false - } - - // If source has error, invalid. - if err := itr.source.Err(); err != nil { - itr.isInvalid = true - return false - } - - // If source is invalid, invalid. - if !itr.source.Valid() { - itr.isInvalid = true - return false - } - - // If key is end or past it, invalid. - start := itr.start - end := itr.end - key := moveSliceToBytes(itr.source.Key()) - if itr.isReverse { - if start != nil && bytes.Compare(key, start) < 0 { - itr.isInvalid = true - return false - } - } else { - if end != nil && bytes.Compare(end, key) <= 0 { - itr.isInvalid = true - return false - } - } - - // It's valid. - return true -} - -// Key implements Iterator. -func (itr *rocksDBIterator) Key() []byte { - itr.assertIsValid() - return moveSliceToBytes(itr.source.Key()) -} - -// Value implements Iterator. -func (itr *rocksDBIterator) Value() []byte { - itr.assertIsValid() - return moveSliceToBytes(itr.source.Value()) -} - -// Next implements Iterator. -func (itr rocksDBIterator) Next() { - itr.assertIsValid() - if itr.isReverse { - itr.source.Prev() - } else { - itr.source.Next() - } -} - -// Error implements Iterator. -func (itr *rocksDBIterator) Error() error { - return itr.source.Err() -} - -// Close implements Iterator. -func (itr *rocksDBIterator) Close() error { - itr.source.Close() - return nil -} - -func (itr *rocksDBIterator) assertIsValid() { - if !itr.Valid() { - panic("iterator is invalid") - } -} - -// moveSliceToBytes will free the slice and copy out a go []byte -// This function can be applied on *Slice returned from Key() and Value() -// of an Iterator, because they are marked as freed. -func moveSliceToBytes(s *grocksdb.Slice) []byte { - defer s.Free() - if !s.Exists() { - return nil - } - v := make([]byte, len(s.Data())) - copy(v, s.Data()) - return v -} diff --git a/rocksdb_test.go b/rocksdb_test.go deleted file mode 100644 index 4eddbc5..0000000 --- a/rocksdb_test.go +++ /dev/null @@ -1,36 +0,0 @@ -//go:build rocksdb -// +build rocksdb - -package db - -import ( - "fmt" - "os" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestRocksDBBackend(t *testing.T) { - name := fmt.Sprintf("test_%x", randStr(12)) - dir := os.TempDir() - db, err := NewDB(name, RocksDBBackend, dir) - require.NoError(t, err) - defer cleanupDBDir(dir, name) - - _, ok := db.(*RocksDB) - assert.True(t, ok) -} - -func TestRocksDBStats(t *testing.T) { - name := fmt.Sprintf("test_%x", randStr(12)) - dir := os.TempDir() - db, err := NewDB(name, RocksDBBackend, dir) - require.NoError(t, err) - defer cleanupDBDir(dir, name) - - assert.NotEmpty(t, db.Stats()) -} - -// TODO: Add tests for rocksdb diff --git a/tools/Dockerfile b/tools/Dockerfile deleted file mode 100644 index a458304..0000000 --- a/tools/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -# This file defines the container image used to build and test tm-db in CI. -# The CI workflows use the latest tag of cometbft/cometbft-db-testing built -# from these settings. -# -# The jobs defined in the Build & Push workflow will build and update the image -# when changes to this file are merged. If you have other changes that require -# updates here, merge the changes here first and let the image get updated (or -# push a new version manually) before PRs that depend on them. - -FROM golang:1.21 AS build - -ENV LD_LIBRARY_PATH=/usr/local/lib - -RUN apt update \ - && apt install -y \ - libbz2-dev libgflags-dev libsnappy-dev libzstd-dev zlib1g-dev liblz4-dev \ - make tar wget build-essential \ - libleveldb-dev libleveldb1d - -FROM build AS install -ARG ROCKSDB=8.9.1 - -# Install Rocksdb -RUN \ - wget -q https://github.com/facebook/rocksdb/archive/refs/tags/v${ROCKSDB}.tar.gz \ - && tar -zxf v${ROCKSDB}.tar.gz \ - && cd rocksdb-${ROCKSDB} \ - && DEBUG_LEVEL=0 make -j4 shared_lib \ - && make install-shared \ - && ldconfig \ - && cd .. \ - && rm -rf v${ROCKSDB}.tar.gz rocksdb-${ROCKSDB}