Refactor bleve tokenizer usage

dgraph/cmd/bulk/mapper.go

@@ -324,7 +324,7 @@ func (m *mapper) addIndexMapEntries(nq gql.NQuad, de *pb.DirectedEdge) {
 		x.Check(err)
 
 		// Extract tokens.
-		toks, err := tok.BuildTokens(schemaVal.Value, toker)
+		toks, err := tok.BuildTokens(schemaVal.Value, tok.GetLangTokenizer(toker, nq.Lang))
 		x.Check(err)
 
 		// Store index posting.

posting/index.go

@@ -25,8 +25,6 @@ import (
 	"time"
 
 	"golang.org/x/net/trace"
-	"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/status"
 
 	"github.com/dgraph-io/badger"
 	"github.com/dgraph-io/badger/y"
@@ -39,8 +37,6 @@ import (
 	"github.com/dgraph-io/dgraph/x"
 )
 
-const maxBatchSize = 32 * (1 << 20)
-
 var emptyCountParams countParams
 
 // IndexTokens return tokens, without the predicate prefix and index rune.
@@ -53,40 +49,25 @@ func indexTokens(attr, lang string, src types.Val) ([]string, error) {
 	if !schema.State().IsIndexed(attr) {
 		return nil, x.Errorf("Attribute %s is not indexed.", attr)
 	}
-	s := schemaType
-	sv, err := types.Convert(src, s)
+	sv, err := types.Convert(src, schemaType)
 	if err != nil {
 		return nil, err
 	}
 	// Schema will know the mapping from attr to tokenizer.
 	var tokens []string
-	tokenizers := schema.State().Tokenizer(attr)
-	for _, it := range tokenizers {
-		if tok.FtsTokenizerName("") == it.Name() && len(lang) > 0 {
-			newTokenizer, ok := tok.GetTokenizer(tok.FtsTokenizerName(lang))
-			if ok {
-				it = newTokenizer
-			} else {
-				return nil, status.Errorf(codes.Internal, "Tokenizer not available for language: %s", lang)
-			}
-		}
-		if schemaType == types.StringID {
-			exactTok, ok := tok.GetTokenizer("exact")
-			x.AssertTruef(ok, "Couldn't find exact tokenizer.")
+	for _, it := range schema.State().Tokenizer(attr) {
+		if it.Name() == "exact" && schemaType == types.StringID && len(sv.Value.(string)) > 100 {
 			// Exact index can only be applied for strings so we can safely try to convert Value to
 			// string.
-			if (it.Identifier() == exactTok.Identifier()) && len(sv.Value.(string)) > 100 {
-				glog.Infof("Long term for exact index on predicate: [%s]. "+
-					"Consider switching to hash for better performance.\n", attr)
-			}
+			glog.Infof("Long term for exact index on predicate: [%s]. "+
+				"Consider switching to hash for better performance.\n", attr)
 		}
-		toks, err := tok.BuildTokens(sv.Value, it)
+		toks, err := tok.BuildTokens(sv.Value, tok.GetLangTokenizer(it, lang))
 		if err != nil {
 			return tokens, err
 		}
 		tokens = append(tokens, toks...)
 	}
-
 	return tokens, nil
 }
 

posting/index_test.go

@@ -104,9 +104,9 @@ func TestIndexingMultiLang(t *testing.T) {
 func TestIndexingInvalidLang(t *testing.T) {
 	schema.ParseBytes([]byte("name:string @index(fulltext) ."), 1)
 
-	// there is no tokenizer for "xx" language
+	// tokenizer for "xx" language won't return an error.
 	_, err := indexTokens("name", "xx", types.Val{Tid: types.StringID, Value: []byte("error")})
-	require.Error(t, err)
+	require.NoError(t, err)
 }
 
 func TestIndexingAliasedLang(t *testing.T) {

query/query3_test.go

@@ -1466,7 +1466,7 @@ func TestInvalidStringIndex(t *testing.T) {
 	require.Error(t, err)
 }
 
-func TestValidFulltextIndex(t *testing.T) {
+func TestValidFullTextIndex(t *testing.T) {
 	// no FTS index defined for name
 
 	query := `

schema/schema.go

@@ -147,7 +147,7 @@ func (s *state) IndexedFields() []string {
 func (s *state) Predicates() []string {
 	s.RLock()
 	defer s.RUnlock()
-	out := make([]string, 0, len(s.predicate))
+	var out []string
 	for k := range s.predicate {
 		out = append(out, k)
 	}
@@ -162,26 +162,21 @@ func (s *state) Tokenizer(pred string) []tok.Tokenizer {
 	x.AssertTruef(ok, "schema state not found for %s", pred)
 	var tokenizers []tok.Tokenizer
 	for _, it := range schema.Tokenizer {
-		t, has := tok.GetTokenizer(it)
-		x.AssertTruef(has, "Invalid tokenizer %s", it)
+		t, found := tok.GetTokenizer(it)
+		x.AssertTruef(found, "Invalid tokenizer %s", it)
 		tokenizers = append(tokenizers, t)
 	}
 	return tokenizers
 }
 
 // TokenizerNames returns the tokenizer names for given predicate
 func (s *state) TokenizerNames(pred string) []string {
-	s.RLock()
-	defer s.RUnlock()
-	schema, ok := s.predicate[pred]
-	x.AssertTruef(ok, "schema state not found for %s", pred)
-	var tokenizers []string
-	for _, it := range schema.Tokenizer {
-		t, found := tok.GetTokenizer(it)
-		x.AssertTruef(found, "Tokenizer not found for %s", it)
-		tokenizers = append(tokenizers, t.Name())
+	var names []string
+	tokenizers := s.Tokenizer(pred)
+	for _, t := range tokenizers {
+		names = append(names, t.Name())
 	}
-	return tokenizers
+	return names
 }
 
 // IsReversed returns whether the predicate has reverse edge or not

systest/loader_test.go

@@ -106,7 +106,7 @@ func TestLoaderXidmap(t *testing.T) {
 	}
 
 	expected = `<_:uid1> <age> "13" .
-<_:uid1> <friend> _:uid2711 .
+<_:uid1> <friend> <_:uid2711> .
 <_:uid1> <location> "Wonderland" .
 <_:uid1> <name> "Alice" .
 <_:uid2711> <name> "Bob" .