Skip to content

Commit

Permalink
index the fields with the Sortable() property before doing requests …
Browse files Browse the repository at this point in the history 
…with custom sorts
  • Loading branch information
@hexun80149128
hexun80149128 committed Feb 19, 2024
1 parent f26b747 commit ed6e779
Show file tree
Hide file tree
Showing 6 changed files with 67 additions and 22 deletions.
5 changes: 3 additions & 2 deletions server/runtime_go.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2598,8 +2598,9 @@ func (ri *RuntimeGoInitializer) RegisterSubscriptionNotificationGoogle(fn func(c
return nil
}

func (ri *RuntimeGoInitializer) RegisterStorageIndex(name, collection, key string, fields []string, maxEntries int, indexOnly bool) error {
return ri.storageIndex.CreateIndex(context.Background(), name, collection, key, fields, maxEntries, indexOnly)
// @param sortFields strings with the keys of the storage object whose values are to be indexed and sorted,it must be in fields,and should be string or num.
func (ri *RuntimeGoInitializer) RegisterStorageIndex(name, collection, key string, fields []string, sortFields []string, maxEntries int, indexOnly bool) error {
return ri.storageIndex.CreateIndex(context.Background(), name, collection, key, fields, sortFields, maxEntries, indexOnly)
}

func (ri *RuntimeGoInitializer) RegisterStorageIndexFilter(indexName string, fn func(ctx context.Context, logger runtime.Logger, db *sql.DB, nk runtime.NakamaModule, write *runtime.StorageWrite) bool) error {
Expand Down
17 changes: 13 additions & 4 deletions server/runtime_javascript_init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1020,14 +1020,23 @@ func (im *RuntimeJavascriptInitModule) registerStorageIndex(r *goja.Runtime) fun
panic(r.NewTypeError("expects an array of strings"))
}

idxMaxEntries := int(getJsInt(r, f.Argument(4)))
ownersSortArray := f.Argument(4)
if goja.IsUndefined(ownersSortArray) || goja.IsNull(ownersSortArray) {
panic(r.NewTypeError("expects an array of fields"))
}
sortFields, err := exportToSlice[[]string](ownersSortArray)
if err != nil {
panic(r.NewTypeError("expects an array of strings"))
}

idxMaxEntries := int(getJsInt(r, f.Argument(5)))

indexOnly := false
if !goja.IsUndefined(f.Argument(5)) && !goja.IsNull(f.Argument(5)) {
indexOnly = getJsBool(r, f.Argument(5))
if !goja.IsUndefined(f.Argument(6)) && !goja.IsNull(f.Argument(6)) {
indexOnly = getJsBool(r, f.Argument(6))
}

if err := im.storageIndex.CreateIndex(context.Background(), idxName, idxCollection, idxKey, fields, idxMaxEntries, indexOnly); err != nil {
if err := im.storageIndex.CreateIndex(context.Background(), idxName, idxCollection, idxKey, fields, sortFields, idxMaxEntries, indexOnly); err != nil {
panic(r.NewGoError(fmt.Errorf("Failed to register storage index: %s", err.Error())))
}

Expand Down
16 changes: 13 additions & 3 deletions server/runtime_lua_nakama.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -510,6 +510,7 @@ func (n *RuntimeLuaNakamaModule) registerLeaderboardReset(l *lua.LState) int {
// @param collection(type=string) Collection of storage engine to index objects from.
// @param key(type=string) Key of storage objects to index. Set to empty string to index all objects of collection.
// @param fields(type=table) A table of strings with the keys of the storage object whose values are to be indexed.
// @param sortFields(type=table) A table of strings with the keys of the storage object whose values are to be indexed and sorted,it must be in fields,and should be string or num.
// @param maxEntries(type=int) Maximum number of entries kept in the index.
// @return error(error) An optional error value if an error occurred.
func (n *RuntimeLuaNakamaModule) registerStorageIndex(l *lua.LState) int {
Expand All @@ -525,10 +526,19 @@ func (n *RuntimeLuaNakamaModule) registerStorageIndex(l *lua.LState) int {
}
fields = append(fields, v.String())
})
maxEntries := l.CheckInt(5)
indexOnly := l.OptBool(6, false)
sortFieldsTable := l.CheckTable(5)
sortFields := make([]string, 0, sortFieldsTable.Len())
sortFieldsTable.ForEach(func(k, v lua.LValue) {
if v.Type() != lua.LTString {
l.ArgError(5, "expects each field to be string")
return
}
sortFields = append(sortFields, v.String())
})
maxEntries := l.CheckInt(6)
indexOnly := l.OptBool(7, false)

if err := n.storageIndex.CreateIndex(context.Background(), idxName, collection, key, fields, maxEntries, indexOnly); err != nil {
if err := n.storageIndex.CreateIndex(context.Background(), idxName, collection, key, fields, sortFields, maxEntries, indexOnly); err != nil {
l.RaiseError("failed to create storage index: %s", err.Error())
}

Expand Down
37 changes: 31 additions & 6 deletions server/storage_index.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"encoding/json"
"errors"
"fmt"
"reflect"
"time"

"github.com/blugelabs/bluge"
Expand All @@ -37,7 +38,7 @@ type StorageIndex interface {
Delete(ctx context.Context, objects StorageOpDeletes) (deletes int)
List(ctx context.Context, callerID uuid.UUID, indexName, query string, limit int, order []string) (*api.StorageObjects, error)
Load(ctx context.Context) error
CreateIndex(ctx context.Context, name, collection, key string, fields []string, maxEntries int, indexOnly bool) error
CreateIndex(ctx context.Context, name, collection, key string, fields []string, sortFields []string, maxEntries int, indexOnly bool) error
RegisterFilters(runtime *Runtime)
}

Expand All @@ -47,6 +48,7 @@ type storageIndex struct {
Collection string
Key string
Fields []string
SortFields []string
IndexOnly bool
Index *bluge.Writer
}
Expand Down Expand Up @@ -122,7 +124,7 @@ func (si *LocalStorageIndex) Write(ctx context.Context, objects []*api.StorageOb
}
}

doc, err := si.mapIndexStorageFields(so.UserId, so.Collection, so.Key, so.Version, so.Value, so.PermissionRead, so.PermissionWrite, so.CreateTime.AsTime(), so.UpdateTime.AsTime(), idx.Fields, idx.IndexOnly)
doc, err := si.mapIndexStorageFields(so.UserId, so.Collection, so.Key, so.Version, so.Value, so.PermissionRead, so.PermissionWrite, so.CreateTime.AsTime(), so.UpdateTime.AsTime(), idx.Fields, idx.SortFields, idx.IndexOnly)
if err != nil {
si.logger.Error("Failed to map storage object values to index", zap.Error(err))
continue
Expand Down Expand Up @@ -238,7 +240,7 @@ func (si *LocalStorageIndex) List(ctx context.Context, callerID uuid.UUID, index

searchReq := bluge.NewTopNSearch(limit, parsedQuery)

if len(order) != 0{
if len(order) != 0 {
searchReq.SortBy(order)
}

Expand Down Expand Up @@ -398,7 +400,7 @@ LIMIT $2`
}
}

doc, err := si.mapIndexStorageFields(dbUserID.String(), idx.Collection, dbKey, dbVersion, dbValue, dbRead, dbWrite, dbCreateTime, dbUpdateTime, idx.Fields, idx.IndexOnly)
doc, err := si.mapIndexStorageFields(dbUserID.String(), idx.Collection, dbKey, dbVersion, dbValue, dbRead, dbWrite, dbCreateTime, dbUpdateTime, idx.Fields, idx.SortFields, idx.IndexOnly)
if err != nil {
rows.Close()
si.logger.Error("Failed to map storage object values to index", zap.Error(err))
Expand Down Expand Up @@ -448,7 +450,7 @@ LIMIT $2`
return nil
}

func (si *LocalStorageIndex) mapIndexStorageFields(userID, collection, key, version, value string, read, write int32, createTime, updateTime time.Time, filters []string, indexOnly bool) (*bluge.Document, error) {
func (si *LocalStorageIndex) mapIndexStorageFields(userID, collection, key, version, value string, read, write int32, createTime, updateTime time.Time, filters []string, sortFilters []string, indexOnly bool) (*bluge.Document, error) {
if collection == "" || key == "" || userID == "" {
return nil, errors.New("insufficient fields to create index document id")
}
Expand Down Expand Up @@ -483,6 +485,28 @@ func (si *LocalStorageIndex) mapIndexStorageFields(userID, collection, key, vers
rv.AddField(bluge.NewNumericField("read", float64(read)).StoreValue())
rv.AddField(bluge.NewNumericField("write", float64(write)).StoreValue())

if len(sortFilters) > 0 {
for _, sortKey := range sortFilters {
if v, found := mapValue[sortKey]; found {
val := reflect.ValueOf(v)
if !val.IsValid() {
continue
}
typ := val.Type()
switch typ.Kind() {
case reflect.String:
rv.AddField(bluge.NewKeywordField(sortKey, val.String()).Sortable())
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
rv.AddField(bluge.NewNumericField(sortKey, float64(val.Int())).Sortable())
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
rv.AddField(bluge.NewNumericField(sortKey, float64(val.Uint())).Sortable())
case reflect.Float32, reflect.Float64:
rv.AddField(bluge.NewNumericField(sortKey, val.Float()).Sortable())
}
}
}
}

if !si.config.DisableIndexOnly && indexOnly {
json, err := json.Marshal(mapValue)
if err != nil {
Expand Down Expand Up @@ -587,7 +611,7 @@ func (si *LocalStorageIndex) queryMatchesToDocumentIds(dmi search.DocumentMatchI
return ids, nil
}

func (si *LocalStorageIndex) CreateIndex(ctx context.Context, name, collection, key string, fields []string, maxEntries int, indexOnly bool) error {
func (si *LocalStorageIndex) CreateIndex(ctx context.Context, name, collection, key string, fields []string, sortFields []string, maxEntries int, indexOnly bool) error {
if name == "" {
return errors.New("storage index 'name' must be set")
}
Expand Down Expand Up @@ -615,6 +639,7 @@ func (si *LocalStorageIndex) CreateIndex(ctx context.Context, name, collection,
Collection: collection,
Key: key,
Fields: fields,
SortFields: sortFields,
MaxEntries: maxEntries,
Index: idx,
IndexOnly: indexOnly,
Expand Down
12 changes: 6 additions & 6 deletions server/storage_index_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,12 @@ func TestLocalStorageIndex_Write(t *testing.T) {
t.Fatal(err.Error())
}

if err := storageIdx.CreateIndex(ctx, indexName1, collection1, key, []string{"one", "two"}, maxEntries1, false); err != nil {
if err := storageIdx.CreateIndex(ctx, indexName1, collection1, key, []string{"one", "two"}, []string{}, maxEntries1, false); err != nil {
t.Fatal(err.Error())
}

// Matches all keys
if err := storageIdx.CreateIndex(ctx, indexName2, collection1, "", []string{"three"}, maxEntries2, false); err != nil {
if err := storageIdx.CreateIndex(ctx, indexName2, collection1, "", []string{"three"}, []string{}, maxEntries2, false); err != nil {
t.Fatal(err.Error())
}

Expand Down Expand Up @@ -336,7 +336,7 @@ func TestLocalStorageIndex_List(t *testing.T) {
t.Fatal(err.Error())
}

if err := storageIdx.CreateIndex(ctx, indexName, collection, key, []string{"one", "two", "three"}, maxEntries, true); err != nil {
if err := storageIdx.CreateIndex(ctx, indexName, collection, key, []string{"one", "two", "three"}, []string{}, maxEntries, true); err != nil {
t.Fatal(err.Error())
}

Expand Down Expand Up @@ -431,8 +431,8 @@ func TestLocalStorageIndex_List(t *testing.T) {
if err != nil {
t.Fatal(err.Error())
}

if err := storageIdx.CreateIndex(ctx, indexName, collection, key, []string{"one", "two", "three", "sort"}, maxEntries, true); err != nil {
if err := storageIdx.CreateIndex(ctx, indexName, collection, key, []string{"one", "two", "three", "sort"}, []string{"sort"}, maxEntries, true); err != nil {
t.Fatal(err.Error())
}

Expand Down Expand Up @@ -532,7 +532,7 @@ func TestLocalStorageIndex_Delete(t *testing.T) {
t.Fatal(err.Error())
}

if err := storageIdx.CreateIndex(ctx, indexName, collection, "", []string{"one"}, maxEntries, false); err != nil {
if err := storageIdx.CreateIndex(ctx, indexName, collection, "", []string{"one"}, []string{}, maxEntries, false); err != nil {
t.Fatal(err.Error())
}

Expand Down
2 changes: 1 addition & 1 deletion vendor/github.com/heroiclabs/nakama-common/runtime/runtime.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit ed6e779

Please sign in to comment.