introduce new scorch index builder

builder.go

@@ -0,0 +1,94 @@
+//  Copyright (c) 2019 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bleve
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/blevesearch/bleve/document"
+	"github.com/blevesearch/bleve/index"
+	"github.com/blevesearch/bleve/index/scorch"
+	"github.com/blevesearch/bleve/mapping"
+)
+
+type builderImpl struct {
+	b index.IndexBuilder
+	m mapping.IndexMapping
+}
+
+func (b *builderImpl) Index(id string, data interface{}) error {
+	if id == "" {
+		return ErrorEmptyID
+	}
+
+	doc := document.NewDocument(id)
+	err := b.m.MapDocument(doc, data)
+	if err != nil {
+		return err
+	}
+	err = b.b.Index(doc)
+	return err
+}
+
+func (b *builderImpl) Close() error {
+	return b.b.Close()
+}
+
+func newBuilder(path string, mapping mapping.IndexMapping, config map[string]interface{}) (Builder, error) {
+	if path == "" {
+		return nil, fmt.Errorf("builder requires path")
+	}
+
+	err := mapping.Validate()
+	if err != nil {
+		return nil, err
+	}
+
+	if config == nil {
+		config = map[string]interface{}{}
+	}
+
+	// the builder does not have an API to interact with internal storage
+	// however we can pass k/v pairs through the config
+	mappingBytes, err := json.Marshal(mapping)
+	if err != nil {
+		return nil, err
+	}
+	config["internal"] = map[string][]byte{
+		string(mappingInternalKey): mappingBytes,
+	}
+
+	// do not use real config, as these are options for the builder,
+	// not the resulting index
+	meta := newIndexMeta(scorch.Name, scorch.Name, map[string]interface{}{})
+	err = meta.Save(path)
+	if err != nil {
+		return nil, err
+	}
+
+	config["path"] = indexStorePath(path)
+
+	b, err := scorch.NewBuilder(config)
+	if err != nil {
+		return nil, err
+	}
+	rv := &builderImpl{
+		b: b,
+		m: mapping,
+	}
+
+	return rv, nil
+}

builder_test.go

@@ -0,0 +1,89 @@
+//  Copyright (c) 2019 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bleve
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func TestBuilder(t *testing.T) {
+	tmpDir, err := ioutil.TempDir("", "bleve-scorch-builder-test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		err = os.RemoveAll(tmpDir)
+		if err != nil {
+			t.Fatalf("error cleaning up test index")
+		}
+	}()
+
+	conf := map[string]interface{}{
+		"batchSize": 2,
+		"mergeMax":  2,
+	}
+	b, err := NewBuilder(tmpDir, NewIndexMapping(), conf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for i := 0; i < 10; i++ {
+		doc := map[string]interface{}{
+			"name": "hello",
+		}
+		err = b.Index(fmt.Sprintf("%d", i), doc)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	err = b.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	idx, err := Open(tmpDir)
+	if err != nil {
+		t.Fatalf("error opening index: %v", err)
+	}
+	defer func() {
+		err = idx.Close()
+		if err != nil {
+			t.Errorf("error closing index: %v", err)
+		}
+	}()
+
+	docCount, err := idx.DocCount()
+	if err != nil {
+		t.Errorf("error checking doc count: %v", err)
+	}
+	if docCount != 10 {
+		t.Errorf("expected doc count to be 10, got %d", docCount)
+	}
+
+	q := NewTermQuery("hello")
+	q.SetField("name")
+	req := NewSearchRequest(q)
+	res, err := idx.Search(req)
+	if err != nil {
+		t.Errorf("error searching index: %v", err)
+	}
+	if res.Total != 10 {
+		t.Errorf("expected 10 search hits, got %d", res.Total)
+	}
+}

index.go

@@ -293,3 +293,17 @@ func Open(path string) (Index, error) {
 func OpenUsing(path string, runtimeConfig map[string]interface{}) (Index, error) {
 	return openIndexUsing(path, runtimeConfig)
 }
+
+// Builder is a limited interface, used to build indexes in an offline mode.
+// Items cannot be updated or deleted, and the caller MUST ensure a document is
+// indexed only once.
+type Builder interface {
+	Index(id string, data interface{}) error
+	Close() error
+}
+
+// NewBuilder creates a builder, which will build an index at the specified path,
+// using the specified mapping and options.
+func NewBuilder(path string, mapping mapping.IndexMapping, config map[string]interface{}) (Builder, error) {
+	return newBuilder(path, mapping, config)
+}

index/index.go

@@ -367,3 +367,10 @@ type OptimizableContext interface {
 type DocValueReader interface {
 	VisitDocValues(id IndexInternalID, visitor DocumentFieldTermVisitor) error
 }
+
+// IndexBuilder is an interface supported by some index schemes
+// to allow direct write-only index building
+type IndexBuilder interface {
+	Index(doc *document.Document) error
+	Close() error
+}