Trigram indexing for regular expressions

Implementation of trigram indexing to avoid scanning/matching
of all values with regular expression.
New tokenizer added (TrigramTokenizer).
Task processing for regexp function changed:

- Trigram index is used to narrow search space.
- Only values returned from trigram index lookup are matched
  with regular expression.

If regexp is to wide-ranging (like: .*), or more than 1000000 values are
returned for trigram, execution is stopped, because of performance
reasons.

Author: Tomasz Zdybał

posting/index.go

@@ -79,6 +79,7 @@ func IndexTokens(attr, lang string, src types.Val) ([]string, error) {
 		}
 		tokens = append(tokens, toks...)
 	}
+
 	return tokens, nil
 }
 

query/query_test.go

@@ -251,7 +251,7 @@ func populateGraph(t *testing.T) {
 		addEdgeToTypedValue(t, "shadow_deep", 24, types.IntID, data.Value.([]byte), nil)
 	}
 
-	// language stuff
+	// Natural Language Processing test data
 	// 0x1001 is uid of interest for language tests
 	addEdgeToLangValue(t, "name", 0x1001, "Badger", "", nil)
 	addEdgeToLangValue(t, "name", 0x1001, "European badger", "en", nil)
@@ -262,6 +262,23 @@ func populateGraph(t *testing.T) {
 	addEdgeToLangValue(t, "name", 0x1001, "Барсук", "ru", nil)
 	addEdgeToLangValue(t, "name", 0x1001, "Blaireau européen", "fr", nil)
 
+	// regex test data
+	// 0x1234 is uid of interest for regex testing
+	addEdgeToValue(t, "name", 0x1234, "Regex Master", nil)
+	nextId := uint64(0x2000)
+	patterns := []string{"mississippi", "missouri", "mission", "missionary",
+		"whissle", "transmission", "zipped", "monosiphonic", "vasopressin", "vapoured",
+		"virtuously", "zurich", "synopsis", "subsensuously",
+		"admission", "commission", "submission", "subcommission", "retransmission", "omission",
+		"permission", "intermission", "dimission", "discommission",
+	}
+
+	for _, p := range patterns {
+		addEdgeToValue(t, "value", nextId, p, nil)
+		addEdgeToUID(t, "pattern", 0x1234, nextId, nil)
+		nextId++
+	}
+
 	addEdgeToValue(t, "name", 240, "Andrea With no friends", nil)
 	time.Sleep(5 * time.Millisecond)
 }
@@ -2531,6 +2548,7 @@ func TestFilterRegexError(t *testing.T) {
 
 func TestFilterRegex1(t *testing.T) {
 	populateGraph(t)
+	posting.CommitLists(10, 1)
 	query := `
     {
       me(id:0x01) {
@@ -2542,13 +2560,13 @@ func TestFilterRegex1(t *testing.T) {
     }
   `
 
-	js := processToFastJSON(t, query)
-	require.JSONEq(t,
-		`{"me":[{"name":"Michonne", "friend":[{"name":"Rick Grimes"},{"name":"Glenn Rhee"},{"name":"Daryl Dixon"}, {"name":"Andrea"}]}]}`, js)
+	_, err := processToFastJsonReq(t, query)
+	require.Error(t, err)
 }
 
 func TestFilterRegex2(t *testing.T) {
 	populateGraph(t)
+	posting.CommitLists(10, 1)
 	query := `
     {
       me(id:0x01) {
@@ -2560,18 +2578,18 @@ func TestFilterRegex2(t *testing.T) {
     }
   `
 
-	js := processToFastJSON(t, query)
-	require.JSONEq(t,
-		`{"me":[{"name":"Michonne", "friend":[{"name":"Rick Grimes"},{"name":"Glenn Rhee"}]}]}`, js)
+	_, err := processToFastJsonReq(t, query)
+	require.Error(t, err)
 }
 
 func TestFilterRegex3(t *testing.T) {
 	populateGraph(t)
+	posting.CommitLists(10, 1)
 	query := `
     {
       me(id:0x01) {
         name
-        friend @filter(regexp(name, "^(Ri)")) {
+        friend @filter(regexp(name, "^Rick")) {
           name
         }
       }
@@ -2583,6 +2601,139 @@ func TestFilterRegex3(t *testing.T) {
 		`{"me":[{"name":"Michonne", "friend":[{"name":"Rick Grimes"}]}]}`, js)
 }
 
+func TestFilterRegex4(t *testing.T) {
+	populateGraph(t)
+	posting.CommitLists(10, 1)
+	query := `
+    {
+      me(id:0x01) {
+        name
+        friend @filter(regexp(name, "((en)|(xo))n")) {
+          name
+        }
+      }
+    }
+  `
+
+	js := processToFastJSON(t, query)
+	require.JSONEq(t,
+		`{"me":[{"name":"Michonne", "friend":[{"name":"Glenn Rhee"},{"name":"Daryl Dixon"} ]}]}`, js)
+}
+
+func TestFilterRegex5(t *testing.T) {
+	populateGraph(t)
+	posting.CommitLists(10, 1)
+	query := `
+    {
+      me(id:0x01) {
+        name
+        friend @filter(regexp(name, "^[a-zA-z]*[^Kk ]?[Nn]ight")) {
+          name
+        }
+      }
+    }
+  `
+
+	js := processToFastJSON(t, query)
+	require.JSONEq(t,
+		`{"me":[{"name":"Michonne"}]}`, js)
+}
+
+func TestFilterRegex6(t *testing.T) {
+	populateGraph(t)
+	posting.CommitLists(10, 1)
+	time.Sleep(100 * time.Millisecond)
+	query := `
+    {
+	  me(id:0x1234) {
+		pattern @filter(regexp(value, "miss((issippi)|(ouri))")) {
+			value
+		}
+      }
+    }
+  `
+
+	js := processToFastJSON(t, query)
+	require.JSONEq(t,
+		`{"me":[{"pattern":[{"value":"mississippi"}, {"value":"missouri"}]}]}`, js)
+}
+
+func TestFilterRegex7(t *testing.T) {
+	populateGraph(t)
+	posting.CommitLists(10, 1)
+	time.Sleep(100 * time.Millisecond)
+	query := `
+    {
+	  me(id:0x1234) {
+		pattern @filter(regexp(value, "[aeiou]mission")) {
+			value
+		}
+      }
+    }
+  `
+
+	js := processToFastJSON(t, query)
+	require.JSONEq(t,
+		`{"me":[{"pattern":[{"value":"omission"}, {"value":"dimission"}]}]}`, js)
+}
+
+func TestFilterRegex8(t *testing.T) {
+	populateGraph(t)
+	posting.CommitLists(10, 1)
+	time.Sleep(100 * time.Millisecond)
+	query := `
+    {
+	  me(id:0x1234) {
+		pattern @filter(regexp(value, "^(trans)?mission")) {
+			value
+		}
+      }
+    }
+  `
+
+	js := processToFastJSON(t, query)
+	require.JSONEq(t,
+		`{"me":[{"pattern":[{"value":"mission"}, {"value":"missionary"}, {"value":"transmission"}]}]}`, js)
+}
+
+func TestFilterRegex9(t *testing.T) {
+	populateGraph(t)
+	posting.CommitLists(10, 1)
+	time.Sleep(100 * time.Millisecond)
+	query := `
+    {
+	  me(id:0x1234) {
+		pattern @filter(regexp(value, "s.{2,5}mission")) {
+			value
+		}
+      }
+    }
+  `
+
+	js := processToFastJSON(t, query)
+	require.JSONEq(t,
+		`{"me":[{"pattern":[{"value":"submission"}, {"value":"subcommission"}, {"value":"discommission"}]}]}`, js)
+}
+
+func TestFilterRegex10(t *testing.T) {
+	populateGraph(t)
+	posting.CommitLists(10, 1)
+	time.Sleep(100 * time.Millisecond)
+	query := `
+    {
+	  me(id:0x1234) {
+		pattern @filter(regexp(value, "[^m]iss")) {
+			value
+		}
+      }
+    }
+  `
+
+	js := processToFastJSON(t, query)
+	require.JSONEq(t,
+		`{"me":[{"pattern":[{"value":"mississippi"}, {"value":"whissle"}]}]}`, js)
+}
+
 func TestToFastJSONFilterUID(t *testing.T) {
 	populateGraph(t)
 	query := `
@@ -5171,7 +5322,8 @@ func TestSchemaBlock1(t *testing.T) {
 		{Predicate: "loc", Type: "geo"}, {Predicate: "alive", Type: "bool"},
 		{Predicate: "shadow_deep", Type: "int"}, {Predicate: "friend", Type: "uid"},
 		{Predicate: "geometry", Type: "geo"}, {Predicate: "alias", Type: "string"},
-		{Predicate: "dob", Type: "date"}, {Predicate: "survival_rate", Type: "float"}}
+		{Predicate: "dob", Type: "date"}, {Predicate: "survival_rate", Type: "float"},
+		{Predicate: "value", Type: "string"}}
 	checkSchemaNodes(t, expected, actual)
 }
 
@@ -5186,7 +5338,7 @@ func TestSchemaBlock2(t *testing.T) {
 	`
 	actual := processSchemaQuery(t, query)
 	expected := []*graphp.SchemaNode{
-		{Predicate: "name", Type: "string", Index: true, Tokenizer: []string{"term", "exact"}}}
+		{Predicate: "name", Type: "string", Index: true, Tokenizer: []string{"term", "exact", "trigram"}}}
 	checkSchemaNodes(t, expected, actual)
 }
 
@@ -5226,12 +5378,12 @@ func TestSchemaBlock5(t *testing.T) {
 	`
 	actual := processSchemaQuery(t, query)
 	expected := []*graphp.SchemaNode{
-		{Predicate: "name", Type: "string", Index: true, Tokenizer: []string{"term", "exact"}}}
+		{Predicate: "name", Type: "string", Index: true, Tokenizer: []string{"term", "exact", "trigram"}}}
 	checkSchemaNodes(t, expected, actual)
 }
 
 const schemaStr = `
-name:string @index(term, exact) .
+name:string @index(term, exact, trigram ) .
 alias:string @index(exact, term, fulltext) .
 dob:date @index .
 film.film.initial_release_date:date @index .
@@ -5243,6 +5395,7 @@ age           : int .
 shadow_deep   : int .
 friend:uid @reverse .
 geometry:geo @index .
+value:string @index(trigram) .
 `
 
 func TestMain(m *testing.M) {
@@ -5267,7 +5420,8 @@ func TestMain(m *testing.M) {
 
 	worker.StartRaftNodes(dir2)
 	// Load schema after nodes have started
-	schema.ParseBytes([]byte(schemaStr), 1)
+	err = schema.ParseBytes([]byte(schemaStr), 1)
+	x.Check(err)
 	defer os.RemoveAll(dir2)
 
 	os.Exit(m.Run())

tok/tok.go

@@ -18,6 +18,7 @@
 package tok
 
 import (
+	"bytes"
 	"encoding/binary"
 	"time"
 
@@ -59,6 +60,7 @@ func init() {
 	RegisterTokenizer(TermTokenizer{})
 	RegisterTokenizer(ExactTokenizer{})
 	RegisterTokenizer(BoolTokenizer{})
+	RegisterTokenizer(TrigramTokenizer{})
 	SetDefault(types.GeoID, "geo")
 	SetDefault(types.IntID, "int")
 	SetDefault(types.FloatID, "float")
@@ -238,6 +240,12 @@ func EncodeGeoTokens(tokens []string) {
 	}
 }
 
+func EncodeRegexTokens(tokens []string) {
+	for i := 0; i < len(tokens); i++ {
+		tokens[i] = encodeToken(tokens[i], TrigramTokenizer{}.Identifier())
+	}
+}
+
 type BoolTokenizer struct{}
 
 func (t BoolTokenizer) Name() string       { return "bool" }
@@ -251,3 +259,27 @@ func (t BoolTokenizer) Tokens(v types.Val) ([]string, error) {
 }
 func (t BoolTokenizer) Identifier() byte { return 0x9 }
 func (t BoolTokenizer) IsSortable() bool { return false }
+
+type TrigramTokenizer struct{}
+
+func (t TrigramTokenizer) Name() string       { return "trigram" }
+func (t TrigramTokenizer) Type() types.TypeID { return types.StringID }
+func (t TrigramTokenizer) Tokens(sv types.Val) ([]string, error) {
+	value, ok := sv.Value.(string)
+	if !ok {
+		return nil, x.Errorf("Trigram indices only supported for string types")
+	}
+	runes := bytes.Runes([]byte(value))
+	l := len(runes) - 2
+	if l > 0 {
+		tokens := make([]string, l)
+		for i := 0; i < l; i++ {
+			trigram := string(runes[i : i+3])
+			tokens[i] = encodeToken(trigram, t.Identifier())
+		}
+		return tokens, nil
+	}
+	return nil, nil
+}
+func (t TrigramTokenizer) Identifier() byte { return 0xA }
+func (t TrigramTokenizer) IsSortable() bool { return false }

tok/tok_test.go

@@ -98,8 +98,33 @@ func TestTermTokenizer(t *testing.T) {
 	val.Value = "Tokenizer works!"
 
 	tokens, err := tokenizer.Tokens(val)
-	require.Nil(t, err)
+	require.NoError(t, err)
 	require.Equal(t, 2, len(tokens))
 	id := tokenizer.Identifier()
 	require.Equal(t, []string{encodeToken("tokenizer", id), encodeToken("works", id)}, tokens)
 }
+
+func TestTrigramTokenizer(t *testing.T) {
+	tokenizer, has := GetTokenizer("trigram")
+	require.True(t, has)
+	require.NotNil(t, tokenizer)
+	val := types.ValueForType(types.StringID)
+	val.Value = "Dgraph rocks!"
+	tokens, err := tokenizer.Tokens(val)
+	require.NoError(t, err)
+	require.Equal(t, 11, len(tokens))
+	id := tokenizer.Identifier()
+	require.Equal(t, []string{
+		encodeToken("Dgr", id),
+		encodeToken("gra", id),
+		encodeToken("rap", id),
+		encodeToken("aph", id),
+		encodeToken("ph ", id),
+		encodeToken("h r", id),
+		encodeToken(" ro", id),
+		encodeToken("roc", id),
+		encodeToken("ock", id),
+		encodeToken("cks", id),
+		encodeToken("ks!", id),
+	}, tokens)
+}