add clone command to gostore

add moss and badger indexers, note: badger indexer has issues blevesearch/bleve/issues/591
change up apis

badger/batch.go

@@ -0,0 +1,44 @@
+package badger
+
+import (
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/dgraph-io/badger"
+)
+
+type Batch struct {
+	store *Store
+	merge *store.EmulatedMerge
+	*badger.Txn
+}
+
+func (b *Batch) Set(key, val []byte) {
+	keyc := make([]byte, len(key))
+	copy(keyc, key)
+
+	valc := make([]byte, len(val))
+	copy(valc, val)
+
+	b.Txn.Set(keyc, valc)
+}
+
+func (b *Batch) Delete(key []byte) {
+	keyc := make([]byte, len(key))
+	copy(keyc, key)
+
+	b.Txn.Delete(keyc)
+}
+
+func (b *Batch) Merge(key, val []byte) {
+	b.merge.Merge(key, val)
+}
+
+func (b *Batch) Reset() {
+	b.merge = store.NewEmulatedMerge(b.store.mo)
+	b.Txn = b.store.db.NewTransaction(true)
+}
+
+func (b *Batch) Close() error {
+	b.merge = nil
+	b.Txn.Discard()
+	return nil
+}

badger/prefix_iterator.go

@@ -0,0 +1,56 @@
+package badger
+
+import (
+	"bytes"
+
+	"github.com/dgraph-io/badger"
+)
+
+type PrefixIterator struct {
+	iterator *badger.Iterator
+	prefix   []byte
+}
+
+func (i *PrefixIterator) Seek(key []byte) {
+	if bytes.Compare(key, i.prefix) < 0 {
+		i.iterator.Seek(i.prefix)
+		return
+	}
+	i.iterator.Seek(key)
+}
+
+func (i *PrefixIterator) Next() {
+	i.iterator.Next()
+}
+
+func (i *PrefixIterator) Current() ([]byte, []byte, bool) {
+	if i.Valid() {
+		return i.Key(), i.Value(), true
+	}
+	return nil, nil, false
+}
+
+func (i *PrefixIterator) Key() []byte {
+	ks := i.iterator.Item().Key()
+	k := make([]byte, len(ks))
+	copy(k, ks)
+
+	return k
+}
+
+func (i *PrefixIterator) Value() []byte {
+	vs, _ := i.iterator.Item().Value()
+	v := make([]byte, len(vs))
+	copy(v, vs)
+
+	return v
+}
+
+func (i *PrefixIterator) Valid() bool {
+	return i.iterator.ValidForPrefix(i.prefix)
+}
+
+func (i *PrefixIterator) Close() error {
+	i.iterator.Close()
+	return nil
+}

badger/range_iterator.go

@@ -0,0 +1,68 @@
+package badger
+
+import (
+	"bytes"
+
+	"github.com/dgraph-io/badger"
+)
+
+type RangeIterator struct {
+	iterator *badger.Iterator
+	start    []byte
+	stop     []byte
+}
+
+func (i *RangeIterator) Seek(key []byte) {
+	if bytes.Compare(key, i.start) < 0 {
+		i.iterator.Seek(i.start)
+		return
+	}
+	i.iterator.Seek(key)
+}
+
+func (i *RangeIterator) Next() {
+	i.iterator.Next()
+}
+
+func (i *RangeIterator) Current() ([]byte, []byte, bool) {
+	if i.Valid() {
+		return i.Key(), i.Value(), true
+	}
+	return nil, nil, false
+}
+
+func (i *RangeIterator) Key() []byte {
+	ks := i.iterator.Item().Key()
+	k := make([]byte, len(ks))
+	copy(k, ks)
+
+	return k
+}
+
+func (i *RangeIterator) Value() []byte {
+	vs, _ := i.iterator.Item().Value()
+	v := make([]byte, len(vs))
+	copy(v, vs)
+
+	return v
+}
+
+func (i *RangeIterator) Valid() bool {
+	if !i.iterator.Valid() {
+		return false
+	}
+
+	if i.stop == nil || len(i.stop) == 0 {
+		return true
+	}
+
+	if bytes.Compare(i.stop, i.iterator.Item().Key()) <= 0 {
+		return false
+	}
+	return true
+}
+
+func (i *RangeIterator) Close() error {
+	i.iterator.Close()
+	return nil
+}

badger/reader.go

@@ -0,0 +1,56 @@
+package badger
+
+import (
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/dgraph-io/badger"
+)
+
+type Reader struct {
+	// you can modify ItrOpts before calling PrefixIterator or PrefixIterator
+	// defaulted to badger.DefaultIteratorOptions by store.Reader()
+	ItrOpts badger.IteratorOptions
+	*badger.Txn
+}
+
+func (r *Reader) Get(k []byte) ([]byte, error) {
+	item, err := r.Txn.Get(k)
+	if err == badger.ErrKeyNotFound {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	vs, err := item.Value()
+	v := make([]byte, len(vs))
+	copy(v, vs)
+
+	return v, err
+}
+
+func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) {
+	return store.MultiGet(r, keys)
+}
+
+func (r *Reader) PrefixIterator(k []byte) store.KVIterator {
+	rv := PrefixIterator{
+		iterator: r.Txn.NewIterator(r.ItrOpts),
+		prefix:   k[:],
+	}
+	rv.iterator.Seek(k)
+	return &rv
+}
+
+func (r *Reader) RangeIterator(start, end []byte) store.KVIterator {
+	rv := RangeIterator{
+		iterator: r.Txn.NewIterator(r.ItrOpts),
+		start:    start[:],
+		stop:     end[:],
+	}
+	rv.iterator.Seek(start)
+	return &rv
+}
+
+func (r *Reader) Close() error {
+	return nil
+}

badger/store.go

@@ -0,0 +1,100 @@
+// MIT LICENSE
+//
+//  Copyright (c) 2017 Fabrice Aneche
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+package badger
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/blevesearch/bleve/registry"
+	"github.com/dgraph-io/badger"
+	"github.com/mgutz/logxi/v1"
+)
+
+const (
+	Name = "badger"
+)
+
+type Store struct {
+	path string
+	db   *badger.DB
+	mo   store.MergeOperator
+}
+
+func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
+	log.Debug("new badger bleve", "config", config, "mo", mo)
+	path, ok := config["path"].(string)
+	if !ok {
+		return nil, fmt.Errorf("must specify path")
+	}
+	if path == "" {
+		return nil, os.ErrInvalid
+	}
+
+	opt := badger.DefaultOptions
+	opt.Dir = path
+	opt.ValueDir = path
+
+	if cdir, ok := config["create_if_missing"].(bool); ok && cdir {
+		if _, err := os.Stat(path); os.IsNotExist(err) {
+			err := os.Mkdir(path, os.FileMode(0700))
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	db, err := badger.Open(opt)
+	if err != nil {
+		return nil, err
+	}
+
+	rv := Store{
+		path: path,
+		db:   db,
+		mo:   mo,
+	}
+	return &rv, nil
+}
+
+func (s *Store) Close() error {
+	return s.db.Close()
+}
+
+func (s *Store) Reader() (store.KVReader, error) {
+	return &Reader{
+		ItrOpts: badger.DefaultIteratorOptions,
+		Txn:     s.db.NewTransaction(false),
+	}, nil
+}
+
+func (s *Store) Writer() (store.KVWriter, error) {
+	return &Writer{
+		s: s,
+	}, nil
+}
+
+func init() {
+	registry.RegisterKVStore(Name, New)
+}