diff --git a/.github/workflows/ci-badger-bank-tests-nightly.yml b/.github/workflows/ci-badger-bank-tests-nightly.yml index 95734786b..513defa01 100644 --- a/.github/workflows/ci-badger-bank-tests-nightly.yml +++ b/.github/workflows/ci-badger-bank-tests-nightly.yml @@ -3,6 +3,7 @@ on: push: branches: - main + - release/v3.2103 schedule: - cron: "0 3 * * *" jobs: diff --git a/.github/workflows/ci-badger-bank-tests.yml b/.github/workflows/ci-badger-bank-tests.yml index be7c0f231..c25f2a366 100644 --- a/.github/workflows/ci-badger-bank-tests.yml +++ b/.github/workflows/ci-badger-bank-tests.yml @@ -3,9 +3,11 @@ on: push: branches: - main + - release/v3.2103 pull_request: branches: - main + - release/v3.2103 schedule: - cron: "*/30 * * * *" jobs: diff --git a/.github/workflows/ci-badger-tests.yml b/.github/workflows/ci-badger-tests.yml index 35c1b416b..e300547aa 100644 --- a/.github/workflows/ci-badger-tests.yml +++ b/.github/workflows/ci-badger-tests.yml @@ -3,9 +3,11 @@ on: push: branches: - main + - release/v3.2103 pull_request: branches: - main + - release/v3.2103 schedule: - cron: "*/30 * * * *" jobs: diff --git a/.github/workflows/ci-golang-lint.yml b/.github/workflows/ci-golang-lint.yml index f3c44b04f..1f30eea94 100644 --- a/.github/workflows/ci-golang-lint.yml +++ b/.github/workflows/ci-golang-lint.yml @@ -3,9 +3,11 @@ on: push: branches: - main + - release/v3.2103 pull_request: branches: - main + - release/v3.2103 schedule: - cron: "*/30 * * * *" jobs: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index 5f1aa266f..000000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,24 +0,0 @@ -# This is a basic workflow to help you get started with Actions - -name: Issue Closer - -# Controls when the action will run. Triggers the workflow on push or pull request -# events but only for the master branch -on: - issues: - types: [ opened, reopened ] - -# A workflow run is made up of one or more jobs that can run sequentially or in parallel -jobs: - # This workflow contains a single job called "build" - build: - # The type of runner that the job will run on - runs-on: ubuntu-latest - - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - - name: Close Issue - uses: peter-evans/close-issue@v1.0.1 - with: - comment: | - **Use [Discuss Issues](https://discuss.dgraph.io/c/issues/badger/37) for reporting issues about this repository.** diff --git a/.go-version b/.go-version new file mode 100644 index 000000000..adc97d8e2 --- /dev/null +++ b/.go-version @@ -0,0 +1 @@ +1.18 diff --git a/.golangci.yml b/.golangci.yml index fecb8644b..81060fb1c 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -3,25 +3,19 @@ run: linters-settings: lll: - line-length: 100 + line-length: 120 linters: disable-all: true enable: - - errcheck - - ineffassign - - gas +# - errcheck +# - ineffassign +# - gas - gofmt - - golint +# - golint - gosimple - govet - lll - - varcheck - - unused - -issues: - exclude-rules: - - linters: - - gosec - text: "G404: " - \ No newline at end of file +# - varcheck +# - unused +# - gosec diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b671868f3..000000000 --- a/.travis.yml +++ /dev/null @@ -1,48 +0,0 @@ -language: go - -go: - - "1.12" - - "1.13" - - tip -os: - - osx -env: - jobs: - - GOARCH=386 - - GOARCH=amd64 - global: - - secure: CRkV2+/jlO0gXzzS50XGxfMS117FNwiVjxNY/LeWq06RKD+dDCPxTJl3JCNe3l0cYEPAglV2uMMYukDiTqJ7e+HI4nh4N4mv6lwx39N8dAvJe1x5ITS2T4qk4kTjuQb1Q1vw/ZOxoQqmvNKj2uRmBdJ/HHmysbRJ1OzCWML3OXdUwJf0AYlJzTjpMfkOKr7sTtE4rwyyQtd4tKH1fGdurgI9ZuFd9qvYxK2qcJhsQ6CNqMXt+7FkVkN1rIPmofjjBTNryzUr4COFXuWH95aDAif19DeBW4lbNgo1+FpDsrgmqtuhl6NAuptI8q/imow2KXBYJ8JPXsxW8DVFj0IIp0RCd3GjaEnwBEbxAyiIHLfW7AudyTS/dJOvZffPqXnuJ8xj3OPIdNe4xY0hWl8Ju2HhKfLOAHq7VadHZWd3IHLil70EiL4/JLD1rNbMImUZisFaA8pyrcIvYYebjOnk4TscwKFLedClRSX1XsMjWWd0oykQtrdkHM2IxknnBpaLu7mFnfE07f6dkG0nlpyu4SCLey7hr5FdcEmljA0nIxTSYDg6035fQkBEAbe7hlESOekkVNT9IZPwG+lmt3vU4ofi6NqNbJecOuSB+h36IiZ9s4YQtxYNnLgW14zjuFGGyT5smc3IjBT7qngDjKIgyrSVoRkY/8udy9qbUgvBeW8= - - -jobs: - allow_failures: - - go: tip - exclude: - # Exclude builds for 386 architecture on go 1.12 and tip - # Since we don't want it to run for 32 bit - - go: "1.12" - env: GOARCH=386 - - go: tip - env: GOARCH=386 - include: - # Define one extra linux build, which we use to run cross - # compiled 32 bit tests - - os: linux - arch: arm64 - go: "1.14" - env: go_32=yes - -notifications: - email: false - slack: - secure: X7uBLWYbuUhf8QFE16CoS5z7WvFR8EN9j6cEectMW6mKZ3vwXGwVXRIPsgUq/606DsQdCCx34MR8MRWYGlu6TBolbSe9y0EP0i46yipPz22YtuT7umcVUbGEyx8MZKgG0v1u/zA0O4aCsOBpGAA3gxz8h3JlEHDt+hv6U8xRsSllVLzLSNb5lwxDtcfEDxVVqP47GMEgjLPM28Pyt5qwjk7o5a4YSVzkfdxBXxd3gWzFUWzJ5E3cTacli50dK4GVfiLcQY2aQYoYO7AAvDnvP+TPfjDkBlUEE4MUz5CDIN51Xb+WW33sX7g+r3Bj7V5IRcF973RiYkpEh+3eoiPnyWyxhDZBYilty3b+Hysp6d4Ov/3I3ll7Bcny5+cYjakjkMH3l9w3gs6Y82GlpSLSJshKWS8vPRsxFe0Pstj6QSJXTd9EBaFr+l1ScXjJv/Sya9j8N9FfTuOTESWuaL1auX4Y7zEEVHlA8SCNOO8K0eTfxGZnC/YcIHsR8rePEAcFxfOYQppkyLF/XvAtnb/LMUuu0g4y2qNdme6Oelvyar1tFEMRtbl4mRCdu/krXBFtkrsfUaVY6WTPdvXAGotsFJ0wuA53zGVhlcd3+xAlSlR3c1QX95HIMeivJKb5L4nTjP+xnrmQNtnVk+tG4LSH2ltuwcZSSczModtcBmRefrk= - -script: >- - if [ $TRAVIS_OS_NAME = "linux" ] && [ $go_32 ]; then - uname -a - GOOS=linux GOARCH=arm go test -v ./... - else - go test -v ./... - # Cross-compile for Plan 9 - GOOS=plan9 go build ./... - fi diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..070b66afa --- /dev/null +++ b/Makefile @@ -0,0 +1,55 @@ +# +# Copyright 2022 Dgraph Labs, Inc. and Contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +USER_ID = $(shell id -u) +HAS_JEMALLOC = $(shell test -f /usr/local/lib/libjemalloc.a && echo "jemalloc") +JEMALLOC_URL = "https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2" + + +.PHONY: all test jemalloc dependency + +test: jemalloc + @echo "Running Badger tests..." + @./test.sh + +jemalloc: + @if [ -z "$(HAS_JEMALLOC)" ] ; then \ + mkdir -p /tmp/jemalloc-temp && cd /tmp/jemalloc-temp ; \ + echo "Downloading jemalloc..." ; \ + curl -s -L ${JEMALLOC_URL} -o jemalloc.tar.bz2 ; \ + tar xjf ./jemalloc.tar.bz2 ; \ + cd jemalloc-5.2.1 ; \ + ./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto'; \ + make ; \ + if [ "$(USER_ID)" -eq "0" ]; then \ + make install ; \ + else \ + echo "==== Need sudo access to install jemalloc" ; \ + sudo make install ; \ + fi \ + fi + +dependency: + @echo "Installing dependencies..." + @sudo apt-get update + @sudo apt-get -y upgrade + @sudo apt-get -y install \ + ca-certificates \ + curl \ + gnupg \ + lsb-release \ + build-essential \ + protobuf-compiler \ diff --git a/README.md b/README.md index aef00efc0..e6a7cf4cb 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,12 @@ -# BadgerDB [![Go Reference](https://pkg.go.dev/badge/github.com/dgraph-io/badger/v3.svg)](https://pkg.go.dev/github.com/dgraph-io/badger/v3) [![Go Report Card](https://goreportcard.com/badge/github.com/dgraph-io/badger)](https://goreportcard.com/report/github.com/dgraph-io/badger) [![Sourcegraph](https://sourcegraph.com/github.com/dgraph-io/badger/-/badge.svg)](https://sourcegraph.com/github.com/dgraph-io/badger?badge) [![Build Status](https://teamcity.dgraph.io/guestAuth/app/rest/builds/buildType:(id:Badger_UnitTests)/statusIcon.svg)](https://teamcity.dgraph.io/viewLog.html?buildTypeId=Badger_UnitTests&buildId=lastFinished&guest=1) ![Appveyor](https://ci.appveyor.com/api/projects/status/github/dgraph-io/badger?branch=master&svg=true) [![Coverage Status](https://coveralls.io/repos/github/dgraph-io/badger/badge.svg?branch=master)](https://coveralls.io/github/dgraph-io/badger?branch=master) +# BadgerDB + +[![Go Reference](https://pkg.go.dev/badge/github.com/dgraph-io/badger/v3.svg)](https://pkg.go.dev/github.com/dgraph-io/badger/v3) +[![Go Report Card](https://goreportcard.com/badge/github.com/dgraph-io/badger/v3)](https://goreportcard.com/report/github.com/dgraph-io/badger/v3) +[![Sourcegraph](https://sourcegraph.com/github.com/dgraph-io/badger/-/badge.svg)](https://sourcegraph.com/github.com/dgraph-io/badger?badge) +[![ci-badger-tests](https://github.com/dgraph-io/badger/actions/workflows/ci-badger-tests.yml/badge.svg)](https://github.com/dgraph-io/badger/actions/workflows/ci-badger-tests.yml) +[![ci-badger-bank-tests](https://github.com/dgraph-io/badger/actions/workflows/ci-badger-bank-tests.yml/badge.svg)](https://github.com/dgraph-io/badger/actions/workflows/ci-badger-bank-tests.yml) +[![ci-golang-lint](https://github.com/dgraph-io/badger/actions/workflows/ci-golang-lint.yml/badge.svg)](https://github.com/dgraph-io/badger/actions/workflows/ci-golang-lint.yml) + ![Badger mascot](images/diggy-shadow.png) @@ -7,9 +15,7 @@ in pure Go. It is the underlying database for [Dgraph](https://dgraph.io), a fast, distributed graph database. It's meant to be a performant alternative to non-Go-based key-value stores like RocksDB. -**Use [Discuss Issues](https://discuss.dgraph.io/c/issues/badger/37) for reporting issues about this repository.** - -## Project Status [March 24, 2020] +## Project Status Badger is stable and is being used to serve data sets worth hundreds of terabytes. Badger supports concurrent ACID transactions with serializable @@ -27,7 +33,9 @@ with v1.0 is v1.6.0. Badger v2.0 was released in Nov 2019 with a new storage format which won't be compatible with all of the v1.x. Badger v2.0 supports compression, encryption and uses a cache to speed up lookup. -The [Changelog] is kept fairly up-to-date. +Badger v3.0 was released in January 2021. This release improves compaction performance. + +Please consult the [Changelog] for more detailed information on releases. For more details on our version naming schema please read [Choosing a version](#choosing-a-version). @@ -51,7 +59,7 @@ For more details on our version naming schema please read [Choosing a version](# ## Getting Started ### Installing -To start using Badger, install Go 1.12 or above. Badger v2 needs go modules. Run the following command to retrieve the library. +To start using Badger, install Go 1.12 or above. Badger v3 needs go modules. From your project, run the following command ```sh $ go get github.com/dgraph-io/badger/v3 @@ -60,13 +68,14 @@ This will retrieve the library. #### Installing Badger Command Line Tool -Download and extract the latest Badger DB release from https://github.com/dgraph-io/badger/releases and then run the following commands. +Badger provides a CLI tool which can perform certain operations like offline backup/restore. To install the Badger CLI, +retrieve the repository and checkout the desired version. Then run ```sh -$ cd badger-/badger -$ go install +$ cd badger +$ go install . ``` -This will install the badger command line utility into your $GOBIN path. +This will install the badger command line utility into your $GOBIN path. #### Choosing a version @@ -86,9 +95,12 @@ Following these rules: version is the same, therefore the data format on disk is compatible. - v1.6.0 and v2.0.0 are data incompatible as their major version implies, so files created with v1.6.0 will need to be converted into the new format before they can be used by v2.0.0. + - v2.x.x and v3.x.x are data incompatible as their major version implies, so files created with + v2.x.x will need to be converted into the new format before they can be used by v3.0.0. + For a longer explanation on the reasons behind using a new versioning naming schema, you can read -[VERSIONING.md](VERSIONING.md). +[VERSIONING](VERSIONING.md). ## Badger Documentation @@ -99,9 +111,9 @@ Badger Documentation is available at https://dgraph.io/docs/badger ### Blog Posts 1. [Introducing Badger: A fast key-value store written natively in Go](https://open.dgraph.io/post/badger/) -2. [Make Badger crash resilient with ALICE](https://blog.dgraph.io/post/alice/) -3. [Badger vs LMDB vs BoltDB: Benchmarking key-value databases in Go](https://blog.dgraph.io/post/badger-lmdb-boltdb/) -4. [Concurrent ACID Transactions in Badger](https://blog.dgraph.io/post/badger-txn/) +2. [Make Badger crash resilient with ALICE](https://open.dgraph.io/post/alice/) +3. [Badger vs LMDB vs BoltDB: Benchmarking key-value databases in Go](https://open.dgraph.io/post/badger-lmdb-boltdb/) +4. [Concurrent ACID Transactions in Badger](https://open.dgraph.io/post/badger-txn/) ## Design Badger was written with these design goals in mind: @@ -167,6 +179,7 @@ Below is a list of known projects that use Badger: * [Immudb](https://github.com/codenotary/immudb) - Lightweight, high-speed immutable database for systems and applications. * [Usenet Express](https://usenetexpress.com/) - Serving over 300TB of data with Badger. * [gorush](https://github.com/appleboy/gorush) - A push notification server written in Go. +* [0-stor](https://github.com/zero-os/0-stor) - Single device object store. * [Dispatch Protocol](https://github.com/dispatchlabs/disgo) - Blockchain protocol for distributed application data analytics. * [GarageMQ](https://github.com/valinurovam/garagemq) - AMQP server written in Go. * [RedixDB](https://alash3al.github.io/redix/) - A real-time persistent key-value store with the same redis protocol. @@ -189,15 +202,23 @@ Below is a list of known projects that use Badger: * [Terminotes](https://gitlab.com/asad-awadia/terminotes) - Self hosted notes storage and search server - storage powered by BadgerDB * [Pyroscope](https://github.com/pyroscope-io/pyroscope) - Open source confinuous profiling platform built with BadgerDB * [Veri](https://github.com/bgokden/veri) - A distributed feature store optimized for Search and Recommendation tasks. +* [bIter](https://github.com/MikkelHJuul/bIter) - A library and Iterator interface for working with the `badger.Iterator`, simplifying from-to, and prefix mechanics. +* [ld](https://github.com/MikkelHJuul/ld) - (Lean Database) A very simple gRPC-only key-value database, exposing BadgerDB with key-range scanning semantics. +* [Souin](https://github.com/darkweak/Souin) - A RFC compliant HTTP cache with lot of other features based on Badger for the storage. Compatible with all existing reverse-proxies. +* [Xuperchain](https://github.com/xuperchain/xupercore) - A highly flexible blockchain architecture with great transaction performance. +* [m2](https://github.com/qichengzx/m2) - A simple http key/value store based on the raft protocol. +* [chaindb](https://github.com/ChainSafe/chaindb) - A blockchain storage layer used by [Gossamer](https://chainsafe.github.io/gossamer/), a Go client for the [Polkadot Network](https://polkadot.network/). +* [vxdb](https://github.com/vitalvas/vxdb) - Simple schema-less Key-Value NoSQL database with simplest API interface. +* [Opacity](https://github.com/opacity/storage-node) - Backend implementation for the Opacity storage project +* [Vephar](https://github.com/vaccovecrana/vephar) - A minimal key/value store using hashicorp-raft for cluster coordination and Badger for data storage. If you are using Badger in a project please send a pull request to add it to the list. ## Contributing -If you're interested in contributing to Badger see [CONTRIBUTING.md](./CONTRIBUTING.md). +If you're interested in contributing to Badger see [CONTRIBUTING](./CONTRIBUTING.md). ## Contact - Please use [discuss.dgraph.io](https://discuss.dgraph.io) for questions, feature requests and discussions. -- Please use [Github issue tracker](https://github.com/dgraph-io/badger/issues) for filing bugs or feature requests. -- Follow us on Twitter [@dgraphlabs](https://twitter.com/dgraphlabs). - +- Please use [discuss.dgraph.io](https://discuss.dgraph.io) for filing bugs or feature requests. +- Follow us on Twitter [@dgraphlabs](https://twitter.com/dgraphlabs). \ No newline at end of file diff --git a/test.sh b/test.sh index 6922eed0b..15122767e 100755 --- a/test.sh +++ b/test.sh @@ -4,48 +4,27 @@ set -eo pipefail go version +# Check if Github Actions is running +if [ $CI = "true" ]; then + # Enable code coverage + # export because tests run in a subprocess + export covermode="-covermode=atomic" + export coverprofile="-coverprofile=cover_tmp.out" + echo "mode: atomic" >> cover.out +fi + # Run `go list` BEFORE setting GOFLAGS so that the output is in the right # format for grep. # export packages because the test will run in a sub process. export packages=$(go list ./... | grep "github.com/dgraph-io/badger/v3/") -if [[ ! -z "$TEAMCITY_VERSION" ]]; then - export GOFLAGS="-json" -fi - -function InstallJemalloc() { - pushd . - if [ ! -f /usr/local/lib/libjemalloc.a ]; then - USER_ID=`id -u` - JEMALLOC_URL="https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2" - - mkdir -p /tmp/jemalloc-temp && cd /tmp/jemalloc-temp ; - echo "Downloading jemalloc" ; - curl -s -L ${JEMALLOC_URL} -o jemalloc.tar.bz2 ; - tar xjf ./jemalloc.tar.bz2 ; - cd jemalloc-5.2.1 ; - ./configure --with-jemalloc-prefix='je_' ; - make ; - if [ "$USER_ID" -eq "0" ]; then - make install ; - else - echo "==== Need sudo access to install jemalloc" ; - sudo make install ; - fi - fi - popd -} - tags="-tags=jemalloc" -# Ensure that we can compile the binary. +# Compile the Badger binary pushd badger go build -v $tags . popd -# tags="" -InstallJemalloc - # Run the memory intensive tests first. manual() { timeout="-timeout 2m" @@ -53,7 +32,7 @@ manual() { set -e for pkg in $packages; do echo "===> Testing $pkg" - go test $tags -timeout=25m -race $pkg -parallel 16 + go test $tags -timeout=25m $covermode $coverprofile -race -parallel 16 $pkg && write_coverage done echo "==> DONE package tests" @@ -61,10 +40,10 @@ manual() { # Run the special Truncate test. rm -rf p set -e - go test $tags $timeout -run='TestTruncateVlogNoClose$' --manual=true + go test $tags $timeout $covermode $coverprofile -run='TestTruncateVlogNoClose$' --manual=true && write_coverage truncate --size=4096 p/000000.vlog - go test $tags $timeout -run='TestTruncateVlogNoClose2$' --manual=true - go test $tags $timeout -run='TestTruncateVlogNoClose3$' --manual=true + go test $tags $timeout $covermode $coverprofile -run='TestTruncateVlogNoClose2$' --manual=true && write_coverage + go test $tags $timeout $covermode $coverprofile -run='TestTruncateVlogNoClose3$' --manual=true && write_coverage rm -rf p # TODO(ibrahim): Let's make these tests have Manual prefix. @@ -73,14 +52,14 @@ manual() { # TestValueGCManaged # TestDropPrefix # TestDropAllManaged - go test $tags $timeout -run='TestBigKeyValuePairs$' --manual=true - go test $tags $timeout -run='TestPushValueLogLimit' --manual=true - go test $tags $timeout -run='TestKeyCount' --manual=true - go test $tags $timeout -run='TestIteratePrefix' --manual=true - go test $tags $timeout -run='TestIterateParallel' --manual=true - go test $tags $timeout -run='TestBigStream' --manual=true - go test $tags $timeout -run='TestGoroutineLeak' --manual=true - go test $tags $timeout -run='TestGetMore' --manual=true + go test $tags $timeout $covermode $coverprofile -run='TestBigKeyValuePairs$' --manual=true && write_coverage + go test $tags $timeout $covermode $coverprofile -run='TestPushValueLogLimit' --manual=true && write_coverage + go test $tags $timeout $covermode $coverprofile -run='TestKeyCount' --manual=true && write_coverage + go test $tags $timeout $covermode $coverprofile -run='TestIteratePrefix' --manual=true && write_coverage + go test $tags $timeout $covermode $coverprofile -run='TestIterateParallel' --manual=true && write_coverage + go test $tags $timeout $covermode $coverprofile -run='TestBigStream' --manual=true && write_coverage + go test $tags $timeout $covermode $coverprofile -run='TestGoroutineLeak' --manual=true && write_coverage + go test $tags $timeout $covermode $coverprofile -run='TestGetMore' --manual=true && write_coverage echo "==> DONE manual tests" } @@ -91,7 +70,7 @@ root() { echo "==> Running root level tests." set -e - go test $tags -timeout=25m . -v -race -parallel 16 + go test $tags -v -race -parallel=16 -timeout=25m $covermode $coverprofile . && write_coverage echo "==> DONE root level tests" } @@ -110,11 +89,23 @@ stream() { return 1 fi echo "==> DONE stream test" + popd return 0 } -export -f stream -export -f manual -export -f root +write_coverage() { + if [ $CI = "true" ]; then + if [ -f cover_tmp.out ]; then + sed -i '1d' cover_tmp.out + cat cover_tmp.out >> cover.out && rm cover_tmp.out + fi + fi + +} -parallel --halt now,fail=1 --progress --line-buffer ::: stream manual root +# parallel tests currently not working +# parallel --halt now,fail=1 --progress --line-buffer ::: stream manual root +# run tests in sequence +root +stream +manual