Improving clan search route latency with mongoDB text index search (#43)

topfreegames · Aug 20, 2019 · 23e8395 · 23e8395
1 parent 25b2fa0
commit 23e8395
Show file tree

Hide file tree

Showing 7 changed files with 72 additions and 6 deletions.
diff --git a/api/clan_test.go b/api/clan_test.go
@@ -21,7 +21,7 @@ import (
 	"github.com/Pallinder/go-randomdata"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"github.com/satori/go.uuid"
+	uuid "github.com/satori/go.uuid"
 	"github.com/topfreegames/khan/api"
 	"github.com/topfreegames/khan/models"
 	"github.com/topfreegames/khan/testing"
@@ -860,6 +860,10 @@ var _ = Describe("Clan API Handler", func() {
 				testDb, gameID, "clan-apisearch-clan", 10,
 			)
 			Expect(err).NotTo(HaveOccurred())
+
+			err = testing.CreateClanNameTextIndexInMongo(GetTestMongo, gameID)
+			Expect(err).NotTo(HaveOccurred())
+
 			status, body := Get(a, GetGameRoute(player.GameID, "clans/search?term=APISEARCH"))
 
 			Expect(status).To(Equal(http.StatusOK))
@@ -917,6 +921,9 @@ var _ = Describe("Clan API Handler", func() {
 			)
 			Expect(err).NotTo(HaveOccurred())
 
+			err = testing.CreateClanNameTextIndexInMongo(GetTestMongo, gameID)
+			Expect(err).NotTo(HaveOccurred())
+
 			url := "clans/search?term=💩clán-clan-APISEARCH"
 			status, body := Get(a, GetGameRoute(player.GameID, url))
 			Expect(status).To(Equal(http.StatusOK))

diff --git a/bench/clan_test.go b/bench/clan_test.go
@@ -15,6 +15,7 @@ import (
 
 	uuid "github.com/satori/go.uuid"
 	"github.com/topfreegames/khan/models"
+	khanTesting "github.com/topfreegames/khan/testing"
 )
 
 var result *http.Response
@@ -188,6 +189,11 @@ func BenchmarkSearchClan(b *testing.B) {
 
 	b.ResetTimer()
 
+	err = khanTesting.CreateClanNameTextIndexInMongo(getTestMongo, game.Name)
+	if err != nil {
+		panic(err.Error())
+	}
+
 	for i := 0; i < b.N; i++ {
 		route := getRoute(fmt.Sprintf("/games/%s/clans/search?term=%s", game.Name, clans[0].PublicID))
 		res, err := get(route)

diff --git a/bench/helpers.go b/bench/helpers.go
@@ -14,10 +14,25 @@ import (
 	"io/ioutil"
 	"net/http"
 
-	"github.com/satori/go.uuid"
+	uuid "github.com/satori/go.uuid"
+	"github.com/spf13/viper"
+	"github.com/topfreegames/extensions/mongo"
+	"github.com/topfreegames/extensions/mongo/interfaces"
 	"github.com/topfreegames/khan/models"
 )
 
+func getTestMongo() (interfaces.MongoDB, error) {
+	config := viper.New()
+	config.SetConfigType("yaml")
+	config.SetConfigFile("../config/perf.yaml")
+	err := config.ReadInConfig()
+	if err != nil {
+		return nil, err
+	}
+	client, err := mongo.NewClient("mongodb", config)
+	return client.MongoDB, err
+}
+
 func getRoute(url string) string {
 	return fmt.Sprintf("http://localhost:8888%s", url)
 }

diff --git a/config/perf.yaml b/config/perf.yaml
@@ -9,6 +9,7 @@ mongodb:
   enabled: true
   url: mongodb://localhost:27017
   databaseName: "khan"
+  database: "khan"
   collectionTemplate: "clans_%s"
 
 search:

diff --git a/models/clan.go b/models/clan.go
@@ -885,12 +885,13 @@ func SearchClan(
 		return clans, nil
 	}
 
+	textSearchTerm := fmt.Sprintf(`"%s"`, term)
+	projection := bson.M{"textSearchScore": bson.M{"$meta": "textScore"}}
 	cmd := bson.D{
 		{Name: "find", Value: fmt.Sprintf("clans_%s", gameID)},
-		{Name: "filter", Value: bson.M{"name": &bson.RegEx{Pattern: term, Options: "i"}}},
-		{Name: "sort", Value: bson.D{
-			{Name: "_id", Value: 1},
-		}},
+		{Name: "filter", Value: bson.M{"$text": bson.M{"$search": textSearchTerm}}},
+		{Name: "projection", Value: projection},
+		{Name: "sort", Value: projection},
 		{Name: "limit", Value: pageSize},
 		{Name: "batchSize", Value: pageSize},
 		{Name: "singleBatch", Value: true},

diff --git a/models/clan_test.go b/models/clan_test.go
@@ -18,6 +18,7 @@ import (
 	uuid "github.com/satori/go.uuid"
 	"github.com/topfreegames/extensions/mongo/interfaces"
 	. "github.com/topfreegames/khan/models"
+	"github.com/topfreegames/khan/testing"
 	"github.com/topfreegames/khan/util"
 
 	"github.com/Pallinder/go-randomdata"
@@ -1069,12 +1070,16 @@ var _ = Describe("Clan Model", func() {
 			})
 
 			It("Should return clan by search term", func() {
+				err := testing.CreateClanNameTextIndexInMongo(GetTestMongo, player.GameID)
+				Expect(err).NotTo(HaveOccurred())
 				clans, err := SearchClan(testDb, testMongo, player.GameID, "SEARCH", 10)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(len(clans)).To(Equal(10))
 			})
 
 			It("Should return clan by unicode search term", func() {
+				err := testing.CreateClanNameTextIndexInMongo(GetTestMongo, player.GameID)
+				Expect(err).NotTo(HaveOccurred())
 				clans, err := SearchClan(testDb, testMongo, player.GameID, "💩clán", 10)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(len(clans)).To(Equal(10))
@@ -1097,6 +1102,8 @@ var _ = Describe("Clan Model", func() {
 			})
 
 			It("Should return empty list if search term is not found", func() {
+				err := testing.CreateClanNameTextIndexInMongo(GetTestMongo, player.GameID)
+				Expect(err).NotTo(HaveOccurred())
 				clans, err := SearchClan(testDb, testMongo, player.GameID, "qwfjur", 10)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(len(clans)).To(Equal(0))

diff --git a/testing/helpers.go b/testing/helpers.go
@@ -0,0 +1,29 @@
+package testing
+
+import (
+	"fmt"
+
+	"github.com/globalsign/mgo/bson"
+	"github.com/topfreegames/extensions/mongo/interfaces"
+)
+
+// CreateClanNameTextIndexInMongo creates the necessary text index for clan search in mongo
+func CreateClanNameTextIndexInMongo(getTestMongo func() (interfaces.MongoDB, error), gameID string) error {
+	mongo, err := getTestMongo()
+	if err != nil {
+		return err
+	}
+
+	cmd := bson.D{
+		{Name: "createIndexes", Value: fmt.Sprintf("clans_%s", gameID)},
+		{Name: "indexes", Value: []interface{}{
+			bson.M{
+				"key": bson.M{
+					"name": "text",
+				},
+				"name": fmt.Sprintf("clans_%s_name_text_index", gameID),
+			},
+		}},
+	}
+	return mongo.Run(cmd, nil)
+}