Integrate badger. Remove RocksDB

Author: Pawan Rawal

.travis.yml

@@ -3,16 +3,13 @@ matrix:
     - os: osx
   include:
     - os: linux
-      compiler: clang-3.9
       go: 1.8
       language: go
-      env: CC=clang-3.9 CXX=clang++-3.9;
     - os: osx
-      compiler: clang
       language: go
       go: 1.8
-      env: CC=clang CXX=clang++;
 
+install: contrib/install.sh
 git:
   depth: 1000
 dist: trusty
@@ -22,14 +19,7 @@ notifications:
     secure: uezT1E7kSVRDC2El9g5syXb1I4JrUUJHW4PXAf0IhXDFqS1ewkrNMurbkltrsJp0OtH7r2rzewuZsxjZ8pz0D+9YPorQzCPUgSqHuYnplh5WJMiaImXVvyApOpVEIm/0mFitrhfGjtMDXegRw1Rk95ujKNokulCSlLqGT1GSKQXmRjBgPYPqPmWQUbzkvwgL6NcMODv1GLyslGGTVKY/WiSjYkiNsQbDjxED3w5rAoX1FG3pvcHAXGxSWbTEk2VzvjGuNwflK92FZZXS/NxXPdEa4lPjlFBRRTHWwEXS4qbaGrdQNJ1DFI/QWrKtw6sHpqt22ovXEk4qkVthaf6Yq8YxoBQ80ajY6gw72ZMXreN3AaHCi7thNKtj/v2e1qlxggNCA8WDnXQmQ0iCg+BYD63LsVbJqlAGq151GGGJXSTcci1DhsuT3JeYx2Vd+wIHNTuYRqFJACnA45KRyExejyzdAYvdBgMnbMbtKctKtFthB5g5P1evBvx/PmZxpXPxIhLKjlCwZgutfR2jy1YSFjIHDR3hBJGsPj1z0XLo36kTOUM/CpTzbNFe2xr3dZrNH/LwWiYO9NBeCHc6kCi6DKi7X7bvnlpk8tazHep7VDQputNWgwEu4flld8jy7QRnIDVIj+4qwT+YhUDG/2lPJ5/Agxjq7rGgkIpPc8Kd8sQ=
 addons:
   apt:
-    sources:
-    - ubuntu-toolchain-r-test
-    - llvm-toolchain-precise-3.9
     packages:
-    - clang-3.9
-    - libbz2-dev
-    - libsnappy-dev
-    - zlib1g-dev
     - jq
     - graphviz
   hosts:
@@ -45,8 +35,6 @@ cache:
   - "$HOME/build"
   - $HOME/Library/Caches/Homebrew
 
-install:
-- bash contrib/build-rocksdb.sh $HOME/build
 before_script:
 - go get github.com/mattn/goveralls
 script:

cmd/dgraph/main.go

@@ -44,6 +44,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/dgraph-io/badger/badger"
 	"golang.org/x/net/context"
 	"golang.org/x/net/trace"
 	"google.golang.org/grpc"
@@ -55,7 +56,6 @@ import (
 	"github.com/dgraph-io/dgraph/query"
 	"github.com/dgraph-io/dgraph/rdf"
 	"github.com/dgraph-io/dgraph/schema"
-	"github.com/dgraph-io/dgraph/store"
 	"github.com/dgraph-io/dgraph/tok"
 	"github.com/dgraph-io/dgraph/types"
 	"github.com/dgraph-io/dgraph/worker"
@@ -1012,8 +1012,11 @@ func main() {
 
 	// All the writes to posting store should be synchronous. We use batched writers
 	// for posting lists, so the cost of sync writes is amortized.
-	ps, err := store.NewSyncStore(*postingDir)
-	x.Checkf(err, "Error initializing postings store")
+	opt := badger.DefaultOptions
+	opt.SyncWrites = true
+	opt.Dir = *postingDir
+	ps, err := badger.NewKV(&opt)
+	x.Checkf(err, "Error while creating badger KV posting store")
 	defer ps.Close()
 
 	x.Check(group.ParseGroupConfig(*gconf))

cmd/dgraph/main_test.go

@@ -29,13 +29,13 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	"github.com/dgraph-io/badger/badger"
 	"github.com/dgraph-io/dgraph/gql"
 	"github.com/dgraph-io/dgraph/group"
 	"github.com/dgraph-io/dgraph/posting"
 	"github.com/dgraph-io/dgraph/protos"
 	"github.com/dgraph-io/dgraph/query"
 	"github.com/dgraph-io/dgraph/schema"
-	"github.com/dgraph-io/dgraph/store"
 	"github.com/dgraph-io/dgraph/types"
 	"github.com/dgraph-io/dgraph/worker"
 	"github.com/dgraph-io/dgraph/x"
@@ -57,16 +57,16 @@ var m = `
 	}
 `
 
-func prepare() (dir1, dir2 string, ps *store.Store, rerr error) {
+func prepare() (dir1, dir2 string, ps *badger.KV, rerr error) {
 	var err error
 	dir1, err = ioutil.TempDir("", "storetest_")
 	if err != nil {
 		return "", "", nil, err
 	}
-	ps, err = store.NewStore(dir1)
-	if err != nil {
-		return "", "", nil, err
-	}
+	opt := badger.DefaultOptions
+	opt.Dir = dir1
+	ps, err = badger.NewKV(&opt)
+	x.Check(err)
 
 	dir2, err = ioutil.TempDir("", "wal_")
 	if err != nil {
@@ -730,12 +730,12 @@ func TestSchemaConversion(t *testing.T) {
 	require.JSONEq(t, `{"user":[{"name2":1.5}]}`, output)
 }
 
-var qErr = `		
- 	mutation {		
- 		set {		
- 			<0x0> <name> "Alice" .		
- 		}		
- 	}		
+var qErr = `
+ 	mutation {
+ 		set {
+ 			<0x0> <name> "Alice" .
+ 		}
+ 	}
  `
 
 func TestMutationError(t *testing.T) {
@@ -945,7 +945,8 @@ func TestExpandPred(t *testing.T) {
 }
 func TestMain(m *testing.M) {
 	x.Init()
-	dir1, dir2, _, _ := prepare()
+	dir1, dir2, ps, _ := prepare()
+	defer ps.Close()
 	defer closeAll(dir1, dir2)
 	time.Sleep(5 * time.Second) // Wait for ME to become leader.
 

contrib/build-rocksdb.sh

@@ -15,24 +15,8 @@ if [ -z "$OUT" ]; then
 fi
 rm -f $OUT
 
-if [[ "${TRAVIS_OS_NAME}" == "osx" ]]; then
-  export LIBRARY_PATH=$LIBRARY_PATH:/usr/local/lib;
-  export C_INCLUDE_PATH=$C_INCLUDE_PATH:/usr/local/include;
-else
-  ROCKSDBDIR=$BUILD/rocksdb-5.1.4
-
-  # build flags needed for rocksdb
-  export CGO_CPPFLAGS="-I${ROCKSDBDIR}/include"
-  export CGO_LDFLAGS="-L${ROCKSDBDIR}"
-  export LD_LIBRARY_PATH="${ROCKSDBDIR}:${LD_LIBRARY_PATH}"
-fi
-
 set -e
 
-# Lets install the dependencies that are not vendored in anymore.
-go get -d golang.org/x/net/context
-go get -d google.golang.org/grpc/...
-
 pushd $SRC &> /dev/null
 
 # create coverage output

contrib/cover.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# Lets install the dependencies that are not vendored in anymore.
+go get -d golang.org/x/net/context
+go get -d google.golang.org/grpc/...
+go get -u github.com/dgraph-io/badger/...
+
+

contrib/install.sh

@@ -22,18 +22,6 @@ ls -la goldendata.rdf.gz
 benchmark=$(pwd)
 popd &> /dev/null
 
-if [[ "${TRAVIS_OS_NAME}" == "osx" ]]; then
-  export LIBRARY_PATH=$LIBRARY_PATH:/usr/local/lib;
-  export C_INCLUDE_PATH=$C_INCLUDE_PATH:/usr/local/include;
-else
-  ROCKSDBDIR=$BUILD/rocksdb-5.1.4
-
-  # build flags needed for rocksdb
-  export CGO_CPPFLAGS="-I${ROCKSDBDIR}/include"
-  export CGO_LDFLAGS="-L${ROCKSDBDIR}"
-  export LD_LIBRARY_PATH="${ROCKSDBDIR}:${LD_LIBRARY_PATH}"
-fi
-
 pushd cmd/dgraph &> /dev/null
 go build .
 ./dgraph -gentlecommit 1.0 &

contrib/loader.sh

@@ -1,25 +1,4 @@
 #!/bin/bash
-
-SRC="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/.."
-
-BUILD=$1
-# If build variable is empty then we set it.
-if [ -z "$1" ]; then
-	  BUILD=$SRC/build
-fi
-
-if [[ "${TRAVIS_OS_NAME}" == "osx" ]]; then
-  export LIBRARY_PATH=$LIBRARY_PATH:/usr/local/lib;
-  export C_INCLUDE_PATH=$C_INCLUDE_PATH:/usr/local/include;
-else
-  ROCKSDBDIR=$BUILD/rocksdb-5.1.4
-
-  # build flags needed for rocksdb
-  export CGO_CPPFLAGS="-I${ROCKSDBDIR}/include"
-  export CGO_LDFLAGS="-L${ROCKSDBDIR}"
-  export LD_LIBRARY_PATH="${ROCKSDBDIR}:${LD_LIBRARY_PATH}"
-fi
-
 set -e
 
 pushd cmd/dgraph &> /dev/null

contrib/queries.sh

@@ -48,12 +48,12 @@ uiDir="main.uiDir"
 echo -e "\033[1;33mBuilding binaries\033[0m"
 echo "dgraph"
 cd $dgraph_cmd/dgraph && \
-   go build -v -tags=embed -ldflags \
+   go build -v -ldflags \
    "-X $release=$release_version -X $branch=$gitBranch -X $commitSHA1=$lastCommitSHA1 -X '$commitTime=$lastCommitTime' -X $uiDir=$ui" .;
 echo "dgraphloader"
 echo $release
 cd $dgraph_cmd/dgraphloader && \
-   go build -v -tags=embed -ldflags \
+   go build -v -ldflags \
    "-X $release=$release_version -X $branch=$gitBranch -X $commitSHA1=$lastCommitSHA1 -X '$commitTime=$lastCommitTime'" .;
 
 echo -e "\n\033[1;33mCopying binaries to tmp folder\033[0m"

contrib/releases/build.sh

@@ -10,20 +10,12 @@ if [ -z "$1" ]; then
         BUILD=$SRC/build
 fi
 
-ROCKSDBDIR=$BUILD/rocksdb-5.1.4
-
 set -e
 
 pushd $BUILD &> /dev/null
 benchmark=$(pwd)/benchmarks/data
 popd &> /dev/null
 
-# build flags needed for rocksdb
-
-export CGO_CPPFLAGS="-I${ROCKSDBDIR}/include"
-export CGO_LDFLAGS="-L${ROCKSDBDIR}"
-export LD_LIBRARY_PATH="${ROCKSDBDIR}:${LD_LIBRARY_PATH}"
-
 pushd cmd/dgraph &> /dev/null
 go build .
 ./dgraph --p ~/dgraph/p0 --w ~/dgraph/w0 &