Add support to custom sorting in storage index search (#1168)

CHANGELOG.md

@@ -6,6 +6,7 @@ The format is based on [keep a changelog](http://keepachangelog.com) and this pr
 ## [Unreleased]
 ### Added
 - Add runtime support for registering a shutdown hook function.
+- Add support to custom sorting in storage index search.
 
 ### Changed
 - When a user is blocked, any DM streams between the blocker and blocked user are torn down.

go.mod

@@ -16,7 +16,7 @@ require (
 	github.com/gorilla/mux v1.8.1
 	github.com/gorilla/websocket v1.5.1
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1
-	github.com/heroiclabs/nakama-common v1.31.1-0.20240513165140-95adfb20e9e3
+	github.com/heroiclabs/nakama-common v1.31.1-0.20240523162557-752591b5e150
 	github.com/jackc/pgconn v1.14.1
 	github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa
 	github.com/jackc/pgtype v1.14.0

go.sum

@@ -155,8 +155,8 @@ github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZH
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1 h1:6UKoz5ujsI55KNpsJH3UwCq3T8kKbZwNZBNPuTTje8U=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.1/go.mod h1:YvJ2f6MplWDhfxiUC3KpyTy76kYUZA4W3pTv/wdKQ9Y=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/heroiclabs/nakama-common v1.31.1-0.20240513165140-95adfb20e9e3 h1:Nrn2Zwss+hA53G5eoGrqtGsjDSXEc5iGbA9XferKe/A=
-github.com/heroiclabs/nakama-common v1.31.1-0.20240513165140-95adfb20e9e3/go.mod h1:Os8XeXGvHAap/p6M/8fQ3gle4eEXDGRQmoRNcPQTjXs=
+github.com/heroiclabs/nakama-common v1.31.1-0.20240523162557-752591b5e150 h1:3H4FQ4tBhs+ri9VhxEbJCf1KXBdtnMRA6j29XxjNzgM=
+github.com/heroiclabs/nakama-common v1.31.1-0.20240523162557-752591b5e150/go.mod h1:Os8XeXGvHAap/p6M/8fQ3gle4eEXDGRQmoRNcPQTjXs=
 github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/influxdata/influxdb v1.7.6/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY=

server/match_common.go

@@ -75,7 +75,7 @@ func IterateBlugeMatches(dmi search.DocumentMatchIterator, loadFields map[string
 	return rv, nil
 }
 
-func BlugeWalkDocument(data interface{}, path []string, doc *bluge.Document) {
+func BlugeWalkDocument(data interface{}, path []string, sortablePaths map[string]bool, doc *bluge.Document) {
 	val := reflect.ValueOf(data)
 	if !val.IsValid() {
 		return
@@ -88,7 +88,7 @@ func BlugeWalkDocument(data interface{}, path []string, doc *bluge.Document) {
 			for _, key := range val.MapKeys() {
 				fieldName := key.String()
 				fieldVal := val.MapIndex(key).Interface()
-				blugeProcessProperty(fieldVal, append(path, fieldName), doc)
+				blugeProcessProperty(fieldVal, append(path, fieldName), sortablePaths, doc)
 			}
 		}
 	case reflect.Struct:
@@ -117,35 +117,35 @@ func BlugeWalkDocument(data interface{}, path []string, doc *bluge.Document) {
 				if fieldName != "" {
 					newpath = append(path, fieldName)
 				}
-				blugeProcessProperty(fieldVal, newpath, doc)
+				blugeProcessProperty(fieldVal, newpath, sortablePaths, doc)
 			}
 		}
 	case reflect.Slice, reflect.Array:
 		for i := 0; i < val.Len(); i++ {
 			if val.Index(i).CanInterface() {
 				fieldVal := val.Index(i).Interface()
-				blugeProcessProperty(fieldVal, path, doc)
+				blugeProcessProperty(fieldVal, path, sortablePaths, doc)
 			}
 		}
 	case reflect.Ptr:
 		ptrElem := val.Elem()
 		if ptrElem.IsValid() && ptrElem.CanInterface() {
-			blugeProcessProperty(ptrElem.Interface(), path, doc)
+			blugeProcessProperty(ptrElem.Interface(), path, sortablePaths, doc)
 		}
 	case reflect.String:
-		blugeProcessProperty(val.String(), path, doc)
+		blugeProcessProperty(val.String(), path, sortablePaths, doc)
 	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-		blugeProcessProperty(float64(val.Int()), path, doc)
+		blugeProcessProperty(float64(val.Int()), path, sortablePaths, doc)
 	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
-		blugeProcessProperty(float64(val.Uint()), path, doc)
+		blugeProcessProperty(float64(val.Uint()), path, sortablePaths, doc)
 	case reflect.Float32, reflect.Float64:
-		blugeProcessProperty(float64(val.Float()), path, doc)
+		blugeProcessProperty(float64(val.Float()), path, sortablePaths, doc)
 	case reflect.Bool:
-		blugeProcessProperty(val.Bool(), path, doc)
+		blugeProcessProperty(val.Bool(), path, sortablePaths, doc)
 	}
 }
 
-func blugeProcessProperty(property interface{}, path []string, doc *bluge.Document) {
+func blugeProcessProperty(property interface{}, path []string, sortablePaths map[string]bool, doc *bluge.Document) {
 	pathString := strings.Join(path, ".")
 
 	propertyValue := reflect.ValueOf(property)
@@ -163,51 +163,75 @@ func blugeProcessProperty(property interface{}, path []string, doc *bluge.Docume
 		parsedDateTime, err := blugeParseDateTime(propertyValueString)
 		if err != nil {
 			// index as text
-			doc.AddField(bluge.NewKeywordField(pathString, propertyValueString))
+			field := bluge.NewKeywordField(pathString, propertyValueString)
+			if sortablePaths[pathString] {
+				field.Sortable()
+			}
+			doc.AddField(field)
 		} else {
 			// index as datetime
-			doc.AddField(bluge.NewDateTimeField(pathString, parsedDateTime))
+			field := bluge.NewDateTimeField(pathString, parsedDateTime)
+			if sortablePaths[pathString] {
+				field.Sortable()
+			}
+			doc.AddField(field)
 		}
 
 	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
-		blugeProcessProperty(float64(propertyValue.Int()), path, doc)
+		blugeProcessProperty(float64(propertyValue.Int()), path, sortablePaths, doc)
 		return
 	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
-		blugeProcessProperty(float64(propertyValue.Uint()), path, doc)
+		blugeProcessProperty(float64(propertyValue.Uint()), path, sortablePaths, doc)
 		return
 	case reflect.Float64, reflect.Float32:
 		propertyValFloat := propertyValue.Float()
 
 		// automatic indexing behavior
-		doc.AddField(bluge.NewNumericField(pathString, propertyValFloat))
+		field := bluge.NewNumericField(pathString, propertyValFloat)
+		if sortablePaths[pathString] {
+			field.Sortable()
+		}
+		doc.AddField(field)
 
 	case reflect.Bool:
 		propertyValBool := propertyValue.Bool()
 
 		// automatic indexing behavior
 		if propertyValBool {
-			doc.AddField(bluge.NewKeywordField(pathString, "T"))
+			field := bluge.NewKeywordField(pathString, "T")
+			if sortablePaths[pathString] {
+				field.Sortable()
+			}
+			doc.AddField(field)
 		} else {
-			doc.AddField(bluge.NewKeywordField(pathString, "F"))
+			field := bluge.NewKeywordField(pathString, "F")
+			if sortablePaths[pathString] {
+				field.Sortable()
+			}
+			doc.AddField(field)
 		}
 
 	case reflect.Struct:
 		switch property := property.(type) {
 		case time.Time:
 			// don't descend into the time struct
-			doc.AddField(bluge.NewDateTimeField(pathString, property))
+			field := bluge.NewDateTimeField(pathString, property)
+			if sortablePaths[pathString] {
+				field.Sortable()
+			}
+			doc.AddField(field)
 
 		default:
-			BlugeWalkDocument(property, path, doc)
+			BlugeWalkDocument(property, path, sortablePaths, doc)
 		}
 	case reflect.Map, reflect.Slice:
-		BlugeWalkDocument(property, path, doc)
+		BlugeWalkDocument(property, path, sortablePaths, doc)
 	case reflect.Ptr:
 		if !propertyValue.IsNil() {
-			BlugeWalkDocument(property, path, doc)
+			BlugeWalkDocument(property, path, sortablePaths, doc)
 		}
 	default:
-		BlugeWalkDocument(property, path, doc)
+		BlugeWalkDocument(property, path, sortablePaths, doc)
 	}
 }
 

server/match_registry.go

@@ -885,7 +885,7 @@ func MapMatchIndexEntry(id string, in *MatchIndexEntry) (*bluge.Document, error)
 	rv.AddField(bluge.NewNumericField("create_time", float64(in.CreateTime)).StoreValue())
 
 	if in.Label != nil {
-		BlugeWalkDocument(in.Label, []string{"label"}, rv)
+		BlugeWalkDocument(in.Label, []string{"label"}, map[string]bool{}, rv)
 	}
 
 	return rv, nil

server/matchmaker.go

@@ -1033,7 +1033,7 @@ func MapMatchmakerIndex(id string, in *MatchmakerIndex) (*bluge.Document, error)
 	rv.AddField(bluge.NewNumericField("created_at", float64(in.CreatedAt)).StoreValue())
 
 	if in.Properties != nil {
-		BlugeWalkDocument(in.Properties, []string{"properties"}, rv)
+		BlugeWalkDocument(in.Properties, []string{"properties"}, map[string]bool{}, rv)
 	}
 
 	return rv, nil

server/runtime_go.go

@@ -2604,8 +2604,8 @@ func (ri *RuntimeGoInitializer) RegisterSubscriptionNotificationGoogle(fn func(c
 	return nil
 }
 
-func (ri *RuntimeGoInitializer) RegisterStorageIndex(name, collection, key string, fields []string, maxEntries int, indexOnly bool) error {
-	return ri.storageIndex.CreateIndex(context.Background(), name, collection, key, fields, maxEntries, indexOnly)
+func (ri *RuntimeGoInitializer) RegisterStorageIndex(name, collection, key string, fields []string, sortableFields []string, maxEntries int, indexOnly bool) error {
+	return ri.storageIndex.CreateIndex(context.Background(), name, collection, key, fields, sortableFields, maxEntries, indexOnly)
 }
 
 func (ri *RuntimeGoInitializer) RegisterStorageIndexFilter(indexName string, fn func(ctx context.Context, logger runtime.Logger, db *sql.DB, nk runtime.NakamaModule, write *runtime.StorageWrite) bool) error {

server/runtime_go_nakama.go

@@ -2096,12 +2096,13 @@ func (n *RuntimeGoNakamaModule) StorageDelete(ctx context.Context, deletes []*ru
 // @group storage
 // @summary List storage index entries
 // @param indexName(type=string) Name of the index to list entries from.
-// @param callerId(type=string, optional=true) User ID of the caller, will apply permissions checks of the user. If empty defaults to system user and permissions are bypassed.
+// @param callerId(type=string, optional=true) User ID of the caller, will apply permissions checks of the user. If empty, defaults to system user and permissions are bypassed.
 // @param queryString(type=string) Query to filter index entries.
 // @param limit(type=int) Maximum number of results to be returned.
+// @param order(type=[]string, optional=true) The storage object fields to sort the query results by. The prefix '-' before a field name indicates descending order. All specified fields must be indexed and sortable.
 // @return objects(*api.StorageObjectList) A list of storage objects.
 // @return error(error) An optional error value if an error occurred.
-func (n *RuntimeGoNakamaModule) StorageIndexList(ctx context.Context, callerID, indexName, query string, limit int) (*api.StorageObjects, error) {
+func (n *RuntimeGoNakamaModule) StorageIndexList(ctx context.Context, callerID, indexName, query string, limit int, order []string) (*api.StorageObjects, error) {
 	cid := uuid.Nil
 	if callerID != "" {
 		id, err := uuid.FromString(callerID)
@@ -2119,7 +2120,7 @@ func (n *RuntimeGoNakamaModule) StorageIndexList(ctx context.Context, callerID,
 		return nil, errors.New("limit must be 1-10000")
 	}
 
-	return n.storageIndex.List(ctx, cid, indexName, query, limit)
+	return n.storageIndex.List(ctx, cid, indexName, query, limit, order)
 }
 
 // @group users

server/runtime_javascript_init.go

@@ -1022,14 +1022,23 @@ func (im *RuntimeJavascriptInitModule) registerStorageIndex(r *goja.Runtime) fun
 			panic(r.NewTypeError("expects an array of strings"))
 		}
 
-		idxMaxEntries := int(getJsInt(r, f.Argument(4)))
+		ownersSortArray := f.Argument(4)
+		if goja.IsUndefined(ownersSortArray) || goja.IsNull(ownersSortArray) {
+			panic(r.NewTypeError("expects an array of fields"))
+		}
+		sortableFields, err := exportToSlice[[]string](ownersSortArray)
+		if err != nil {
+			panic(r.NewTypeError("expects an array of strings"))
+		}
+
+		idxMaxEntries := int(getJsInt(r, f.Argument(5)))
 
 		indexOnly := false
-		if !goja.IsUndefined(f.Argument(5)) && !goja.IsNull(f.Argument(5)) {
-			indexOnly = getJsBool(r, f.Argument(5))
+		if !goja.IsUndefined(f.Argument(6)) && !goja.IsNull(f.Argument(6)) {
+			indexOnly = getJsBool(r, f.Argument(6))
 		}
 
-		if err := im.storageIndex.CreateIndex(context.Background(), idxName, idxCollection, idxKey, fields, idxMaxEntries, indexOnly); err != nil {
+		if err := im.storageIndex.CreateIndex(context.Background(), idxName, idxCollection, idxKey, fields, sortableFields, idxMaxEntries, indexOnly); err != nil {
 			panic(r.NewGoError(fmt.Errorf("Failed to register storage index: %s", err.Error())))
 		}
 

server/runtime_javascript_nakama.go

@@ -349,6 +349,7 @@ func (n *runtimeJavascriptNakamaModule) stringToBinary(r *goja.Runtime) func(goj
 // @param indexName(type=string) Name of the index to list entries from.
 // @param queryString(type=string) Query to filter index entries.
 // @param limit(type=int) Maximum number of results to be returned.
+// @param order(type=[]string, optional=true) The storage object fields to sort the query results by. The prefix '-' before a field name indicates descending order. All specified fields must be indexed and sortable.
 // @param callerId(type=string, optional=true) User ID of the caller, will apply permissions checks of the user. If empty defaults to system user and permission checks are bypassed.
 // @return objects(nkruntime.StorageObjectList) A list of storage objects.
 // @return error(error) An optional error value if an error occurred.
@@ -363,17 +364,28 @@ func (n *runtimeJavascriptNakamaModule) storageIndexList(r *goja.Runtime) func(g
 				panic(r.NewTypeError("limit must be 1-10000"))
 			}
 		}
+
+		var err error
+		order := make([]string, 0)
+		orderIn := f.Argument(3)
+		if !goja.IsUndefined(orderIn) && !goja.IsNull(orderIn) {
+			order, err = exportToSlice[[]string](orderIn)
+			if err != nil {
+				panic(r.NewTypeError("expects an array of strings"))
+			}
+		}
+
 		callerID := uuid.Nil
-		if !goja.IsUndefined(f.Argument(3)) && !goja.IsNull(f.Argument(3)) {
-			callerIdStr := getJsString(r, f.Argument(3))
+		if !goja.IsUndefined(f.Argument(4)) && !goja.IsNull(f.Argument(4)) {
+			callerIdStr := getJsString(r, f.Argument(4))
 			cid, err := uuid.FromString(callerIdStr)
 			if err != nil {
 				panic(r.NewTypeError("expects caller id to be valid identifier"))
 			}
 			callerID = cid
 		}
 
-		objectList, err := n.storageIndex.List(n.ctx, callerID, idxName, queryString, int(limit))
+		objectList, err := n.storageIndex.List(n.ctx, callerID, idxName, queryString, int(limit), order)
 		if err != nil {
 			panic(r.NewGoError(fmt.Errorf("failed to lookup storage index: %s", err.Error())))
 		}

server/runtime_lua_nakama.go

@@ -528,6 +528,7 @@ func (n *RuntimeLuaNakamaModule) registerShutdown(l *lua.LState) int {
 // @param collection(type=string) Collection of storage engine to index objects from.
 // @param key(type=string) Key of storage objects to index. Set to empty string to index all objects of collection.
 // @param fields(type=table) A table of strings with the keys of the storage object whose values are to be indexed.
+// @param sortableFields(type=table, optional=true) A table of strings with the keys of the storage object whose values are to be sortable. The keys must exist within the previously specified fields to be indexed.
 // @param maxEntries(type=int) Maximum number of entries kept in the index.
 // @return error(error) An optional error value if an error occurred.
 func (n *RuntimeLuaNakamaModule) registerStorageIndex(l *lua.LState) int {
@@ -543,10 +544,19 @@ func (n *RuntimeLuaNakamaModule) registerStorageIndex(l *lua.LState) int {
 		}
 		fields = append(fields, v.String())
 	})
-	maxEntries := l.CheckInt(5)
-	indexOnly := l.OptBool(6, false)
+	sortFieldsTable := l.CheckTable(5)
+	sortableFields := make([]string, 0, sortFieldsTable.Len())
+	sortFieldsTable.ForEach(func(k, v lua.LValue) {
+		if v.Type() != lua.LTString {
+			l.ArgError(5, "expects each field to be string")
+			return
+		}
+		sortableFields = append(sortableFields, v.String())
+	})
+	maxEntries := l.CheckInt(6)
+	indexOnly := l.OptBool(7, false)
 
-	if err := n.storageIndex.CreateIndex(context.Background(), idxName, collection, key, fields, maxEntries, indexOnly); err != nil {
+	if err := n.storageIndex.CreateIndex(context.Background(), idxName, collection, key, fields, sortableFields, maxEntries, indexOnly); err != nil {
 		l.RaiseError("failed to create storage index: %s", err.Error())
 	}
 
@@ -10122,6 +10132,7 @@ func (n *RuntimeLuaNakamaModule) channelIdBuild(l *lua.LState) int {
 // @param indexName(type=string) Name of the index to list entries from.
 // @param queryString(type=string) Query to filter index entries.
 // @param limit(type=int) Maximum number of results to be returned.
+// @param order(type=[]string, optional=true) The storage object fields to sort the query results by. The prefix '-' before a field name indicates descending order. All specified fields must be indexed and sortable.
 // @param callerId(type=string, optional=true) User ID of the caller, will apply permissions checks of the user. If empty defaults to system user and permission checks are bypassed.
 // @return objects(table) A list of storage objects.
 // @return error(error) An optional error value if an error occurred.
@@ -10133,18 +10144,28 @@ func (n *RuntimeLuaNakamaModule) storageIndexList(l *lua.LState) int {
 		l.ArgError(3, "invalid limit: expects value 1-10000")
 		return 0
 	}
+	orderTable := l.CheckTable(4)
+	order := make([]string, 0, orderTable.Len())
+	orderTable.ForEach(func(k, v lua.LValue) {
+		if v.Type() != lua.LTString {
+			l.ArgError(4, "expects each field to be string")
+			return
+		}
+		order = append(order, v.String())
+	})
+
 	callerID := uuid.Nil
-	callerIDStr := l.OptString(4, "")
+	callerIDStr := l.OptString(5, "")
 	if callerIDStr != "" {
 		cid, err := uuid.FromString(callerIDStr)
 		if err != nil {
-			l.ArgError(4, "expects caller ID to be empty or a valid identifier")
+			l.ArgError(5, "expects caller ID to be empty or a valid identifier")
 			return 0
 		}
 		callerID = cid
 	}
 
-	objectList, err := n.storageIndex.List(l.Context(), callerID, idxName, queryString, limit)
+	objectList, err := n.storageIndex.List(l.Context(), callerID, idxName, queryString, limit, order)
 	if err != nil {
 		l.RaiseError(err.Error())
 		return 0