From 9003ee2f291a87fba654d70ff38786538b5f51d2 Mon Sep 17 00:00:00 2001 From: Somiacao <36264120+Somiacao@users.noreply.github.com> Date: Sat, 12 Nov 2022 14:33:37 +0800 Subject: [PATCH] support OpenTelemetry tracing (#576) --- base/log/log.go | 11 ++ base/log/log_test.go | 7 + client/client.go | 58 +++--- client/client_test.go | 34 ++-- cmd/gorse-in-one/main.go | 6 +- config/config.go | 93 ++++++++++ config/config.toml.template | 17 ++ config/config_test.go | 6 + go.mod | 45 +++-- go.sum | 167 ++++++++++++++--- master/master.go | 18 +- master/rest.go | 148 +++++++++------ master/rest_test.go | 121 ++++++++----- master/tasks.go | 107 ++++++----- master/tasks_test.go | 231 ++++++++++++------------ server/bench_test.go | 25 +-- server/rest.go | 287 +++++++++++++++++++++--------- server/rest_test.go | 114 ++++++------ server/server.go | 36 +++- storage/cache/database.go | 78 +++++--- storage/cache/database_test.go | 110 ++++++------ storage/cache/mongodb.go | 39 ++-- storage/cache/no_database.go | 28 +-- storage/cache/no_database_test.go | 32 ++-- storage/cache/redis.go | 39 ++-- storage/cache/sql.go | 62 ++++--- storage/data/database.go | 83 +++++---- storage/data/database_test.go | 240 +++++++++++++------------ storage/data/mongodb.go | 61 +++---- storage/data/no_database.go | 41 ++--- storage/data/no_database_test.go | 44 ++--- storage/data/redis.go | 79 ++++---- storage/data/sql.go | 108 ++++++----- worker/worker.go | 75 ++++---- worker/worker_test.go | 189 +++++++++++--------- 35 files changed, 1726 insertions(+), 1113 deletions(-) diff --git a/base/log/log.go b/base/log/log.go index 2fdf7e89c..6d2e48c3e 100644 --- a/base/log/log.go +++ b/base/log/log.go @@ -17,6 +17,7 @@ package log import ( "github.com/emicklei/go-restful/v3" "github.com/go-sql-driver/mysql" + "go.opentelemetry.io/otel" "go.uber.org/zap" "net/url" "os" @@ -105,3 +106,13 @@ func RedactDBURL(rawURL string) string { return parsed.String() } } + +func GetErrorHandler() otel.ErrorHandler { + return &errorHandler{} +} + +type errorHandler struct{} + +func (h *errorHandler) Handle(err error) { + Logger().Error("opentelemetry failure", zap.Error(err)) +} diff --git a/base/log/log_test.go b/base/log/log_test.go index 5b4624059..fee2c5015 100644 --- a/base/log/log_test.go +++ b/base/log/log_test.go @@ -17,10 +17,14 @@ package log import ( "github.com/stretchr/testify/assert" "os" + "runtime" "testing" ) func TestSetDevelopmentLogger(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } temp, err := os.MkdirTemp("", "test_gorse") assert.NoError(t, err) // set existed path @@ -41,6 +45,9 @@ func TestSetDevelopmentLogger(t *testing.T) { } func TestSetProductionLogger(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } temp, err := os.MkdirTemp("", "test_gorse") assert.NoError(t, err) // set existed path diff --git a/client/client.go b/client/client.go index e5cee0ee3..6e652e9a1 100644 --- a/client/client.go +++ b/client/client.go @@ -15,11 +15,14 @@ package client import ( + "context" "encoding/json" "fmt" "io" "net/http" "strings" + + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" ) type GorseClient struct { @@ -28,64 +31,65 @@ type GorseClient struct { httpClient http.Client } -func NewGorseClient(EntryPoint, ApiKey string) *GorseClient { +func NewGorseClient(entryPoint, apiKey string) *GorseClient { return &GorseClient{ - entryPoint: EntryPoint, - apiKey: ApiKey, + entryPoint: entryPoint, + apiKey: apiKey, + httpClient: http.Client{Transport: otelhttp.NewTransport(http.DefaultTransport)}, } } -func (c *GorseClient) InsertFeedback(feedbacks []Feedback) (RowAffected, error) { - return request[RowAffected](c, "POST", c.entryPoint+"/api/feedback", feedbacks) +func (c *GorseClient) InsertFeedback(ctx context.Context, feedbacks []Feedback) (RowAffected, error) { + return request[RowAffected](ctx, c, "POST", c.entryPoint+"/api/feedback", feedbacks) } -func (c *GorseClient) ListFeedbacks(feedbackType, userId string) ([]Feedback, error) { - return request[[]Feedback, any](c, "GET", c.entryPoint+fmt.Sprintf("/api/user/"+userId+"/feedback/"+feedbackType), nil) +func (c *GorseClient) ListFeedbacks(ctx context.Context, feedbackType, userId string) ([]Feedback, error) { + return request[[]Feedback, any](ctx, c, "GET", c.entryPoint+fmt.Sprintf("/api/user/"+userId+"/feedback/"+feedbackType), nil) } -func (c *GorseClient) GetRecommend(userId string, category string, n int) ([]string, error) { - return request[[]string, any](c, "GET", c.entryPoint+fmt.Sprintf("/api/recommend/%s/%s?n=%d", userId, category, n), nil) +func (c *GorseClient) GetRecommend(ctx context.Context, userId string, category string, n int) ([]string, error) { + return request[[]string, any](ctx, c, "GET", c.entryPoint+fmt.Sprintf("/api/recommend/%s/%s?n=%d", userId, category, n), nil) } -func (c *GorseClient) SessionRecommend(feedbacks []Feedback, n int) ([]Score, error) { - return request[[]Score](c, "POST", c.entryPoint+fmt.Sprintf("/api/session/recommend?n=%d", n), feedbacks) +func (c *GorseClient) SessionRecommend(ctx context.Context, feedbacks []Feedback, n int) ([]Score, error) { + return request[[]Score](ctx, c, "POST", c.entryPoint+fmt.Sprintf("/api/session/recommend?n=%d", n), feedbacks) } -func (c *GorseClient) GetNeighbors(itemId string, n int) ([]Score, error) { - return request[[]Score, any](c, "GET", c.entryPoint+fmt.Sprintf("/api/item/%s/neighbors?n=%d", itemId, n), nil) +func (c *GorseClient) GetNeighbors(ctx context.Context, itemId string, n int) ([]Score, error) { + return request[[]Score, any](ctx, c, "GET", c.entryPoint+fmt.Sprintf("/api/item/%s/neighbors?n=%d", itemId, n), nil) } -func (c *GorseClient) InsertUser(user User) (RowAffected, error) { - return request[RowAffected](c, "POST", c.entryPoint+"/api/user", user) +func (c *GorseClient) InsertUser(ctx context.Context, user User) (RowAffected, error) { + return request[RowAffected](ctx, c, "POST", c.entryPoint+"/api/user", user) } -func (c *GorseClient) GetUser(userId string) (User, error) { - return request[User, any](c, "GET", c.entryPoint+fmt.Sprintf("/api/user/%s", userId), nil) +func (c *GorseClient) GetUser(ctx context.Context, userId string) (User, error) { + return request[User, any](ctx, c, "GET", c.entryPoint+fmt.Sprintf("/api/user/%s", userId), nil) } -func (c *GorseClient) DeleteUser(userId string) (RowAffected, error) { - return request[RowAffected, any](c, "DELETE", c.entryPoint+fmt.Sprintf("/api/user/%s", userId), nil) +func (c *GorseClient) DeleteUser(ctx context.Context, userId string) (RowAffected, error) { + return request[RowAffected, any](ctx, c, "DELETE", c.entryPoint+fmt.Sprintf("/api/user/%s", userId), nil) } -func (c *GorseClient) InsertItem(item Item) (RowAffected, error) { - return request[RowAffected](c, "POST", c.entryPoint+"/api/item", item) +func (c *GorseClient) InsertItem(ctx context.Context, item Item) (RowAffected, error) { + return request[RowAffected](ctx, c, "POST", c.entryPoint+"/api/item", item) } -func (c *GorseClient) GetItem(itemId string) (Item, error) { - return request[Item, any](c, "GET", c.entryPoint+fmt.Sprintf("/api/item/%s", itemId), nil) +func (c *GorseClient) GetItem(ctx context.Context, itemId string) (Item, error) { + return request[Item, any](ctx, c, "GET", c.entryPoint+fmt.Sprintf("/api/item/%s", itemId), nil) } -func (c *GorseClient) DeleteItem(itemId string) (RowAffected, error) { - return request[RowAffected, any](c, "DELETE", c.entryPoint+fmt.Sprintf("/api/item/%s", itemId), nil) +func (c *GorseClient) DeleteItem(ctx context.Context, itemId string) (RowAffected, error) { + return request[RowAffected, any](ctx, c, "DELETE", c.entryPoint+fmt.Sprintf("/api/item/%s", itemId), nil) } -func request[Response any, Body any](c *GorseClient, method, url string, body Body) (result Response, err error) { +func request[Response any, Body any](ctx context.Context, c *GorseClient, method, url string, body Body) (result Response, err error) { bodyByte, marshalErr := json.Marshal(body) if marshalErr != nil { return result, marshalErr } var req *http.Request - req, err = http.NewRequest(method, url, strings.NewReader(string(bodyByte))) + req, err = http.NewRequestWithContext(ctx, method, url, strings.NewReader(string(bodyByte))) if err != nil { return result, err } diff --git a/client/client_test.go b/client/client_test.go index 0203f4dcf..3f71f1ed1 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -49,9 +49,10 @@ func (suite *GorseClientTestSuite) TearDownSuite() { } func (suite *GorseClientTestSuite) TestFeedback() { + ctx := context.TODO() timestamp := time.Unix(1660459054, 0).UTC().Format(time.RFC3339) userId := "800" - insertFeedbackResp, err := suite.client.InsertFeedback([]Feedback{{ + insertFeedbackResp, err := suite.client.InsertFeedback(ctx, []Feedback{{ FeedbackType: "like", UserId: userId, Timestamp: timestamp, @@ -60,7 +61,7 @@ func (suite *GorseClientTestSuite) TestFeedback() { suite.NoError(err) suite.Equal(1, insertFeedbackResp.RowAffected) - insertFeedbacksResp, err := suite.client.InsertFeedback([]Feedback{{ + insertFeedbacksResp, err := suite.client.InsertFeedback(ctx, []Feedback{{ FeedbackType: "read", UserId: userId, Timestamp: timestamp, @@ -74,7 +75,7 @@ func (suite *GorseClientTestSuite) TestFeedback() { suite.NoError(err) suite.Equal(2, insertFeedbacksResp.RowAffected) - feedbacks, err := suite.client.ListFeedbacks("read", userId) + feedbacks, err := suite.client.ListFeedbacks(ctx, "read", userId) suite.NoError(err) suite.ElementsMatch([]Feedback{ { @@ -92,7 +93,8 @@ func (suite *GorseClientTestSuite) TestFeedback() { } func (suite *GorseClientTestSuite) TestRecommend() { - suite.redis.ZAddArgs(context.Background(), "offline_recommend/100", redis.ZAddArgs{ + ctx := context.TODO() + suite.redis.ZAddArgs(ctx, "offline_recommend/100", redis.ZAddArgs{ Members: []redis.Z{ { Score: 1, @@ -108,7 +110,7 @@ func (suite *GorseClientTestSuite) TestRecommend() { }, }, }) - resp, err := suite.client.GetRecommend("100", "", 10) + resp, err := suite.client.GetRecommend(ctx, "100", "", 10) suite.NoError(err) suite.Equal([]string{"3", "2", "1"}, resp) } @@ -191,7 +193,7 @@ func (suite *GorseClientTestSuite) TestSessionRecommend() { feedbackType := "like" userId := "0" timestamp := time.Unix(1660459054, 0).UTC().Format(time.RFC3339) - resp, err := suite.client.SessionRecommend([]Feedback{ + resp, err := suite.client.SessionRecommend(ctx, []Feedback{ { FeedbackType: feedbackType, UserId: userId, @@ -258,7 +260,7 @@ func (suite *GorseClientTestSuite) TestNeighbors() { }) itemId := "100" - resp, err := suite.client.GetNeighbors(itemId, 3) + resp, err := suite.client.GetNeighbors(ctx, itemId, 3) suite.NoError(err) suite.Equal([]Score{ { @@ -275,29 +277,31 @@ func (suite *GorseClientTestSuite) TestNeighbors() { } func (suite *GorseClientTestSuite) TestUsers() { + ctx := context.TODO() user := User{ UserId: "100", Labels: []string{"a", "b", "c"}, Subscribe: []string{"d", "e"}, Comment: "comment", } - rowAffected, err := suite.client.InsertUser(user) + rowAffected, err := suite.client.InsertUser(ctx, user) suite.NoError(err) suite.Equal(1, rowAffected.RowAffected) - userResp, err := suite.client.GetUser("100") + userResp, err := suite.client.GetUser(ctx, "100") suite.NoError(err) suite.Equal(user, userResp) - deleteAffect, err := suite.client.DeleteUser("100") + deleteAffect, err := suite.client.DeleteUser(ctx, "100") suite.NoError(err) suite.Equal(1, deleteAffect.RowAffected) - _, err = suite.client.GetUser("100") + _, err = suite.client.GetUser(ctx, "100") suite.Equal("100: user not found", err.Error()) } func (suite *GorseClientTestSuite) TestItems() { + ctx := context.TODO() timestamp := time.Unix(1660459054, 0).UTC().Format(time.RFC3339) item := Item{ ItemId: "100", @@ -307,19 +311,19 @@ func (suite *GorseClientTestSuite) TestItems() { Timestamp: timestamp, Comment: "comment", } - rowAffected, err := suite.client.InsertItem(item) + rowAffected, err := suite.client.InsertItem(ctx, item) suite.NoError(err) suite.Equal(1, rowAffected.RowAffected) - itemResp, err := suite.client.GetItem("100") + itemResp, err := suite.client.GetItem(ctx, "100") suite.NoError(err) suite.Equal(item, itemResp) - deleteAffect, err := suite.client.DeleteItem("100") + deleteAffect, err := suite.client.DeleteItem(ctx, "100") suite.NoError(err) suite.Equal(1, deleteAffect.RowAffected) - _, err = suite.client.GetItem("100") + _, err = suite.client.GetItem(ctx, "100") suite.Equal("100: item not found", err.Error()) } diff --git a/cmd/gorse-in-one/main.go b/cmd/gorse-in-one/main.go index 4bae00d9a..195011ac1 100644 --- a/cmd/gorse-in-one/main.go +++ b/cmd/gorse-in-one/main.go @@ -92,9 +92,9 @@ var oneCommand = &cobra.Command{ } fmt.Println() - fmt.Printf(" Dashboard: http://%s:%d/overview\n", conf.Master.HttpHost, conf.Master.HttpPort) - fmt.Printf(" RESTful APIs: http://%s:%d/apidocs\n", conf.Master.HttpHost, conf.Master.HttpPort) - fmt.Printf(" Documentation: https://docs.gorse.io/\n") + fmt.Printf(" Dashboard: http://127.0.0.1:%d/overview\n", conf.Master.HttpPort) + fmt.Printf(" RESTful APIs: http://127.0.0.1:%d/apidocs\n", conf.Master.HttpPort) + fmt.Printf(" Documentation: https://gorse.io/docs\n") fmt.Println() } else { configPath, _ := cmd.PersistentFlags().GetString("config") diff --git a/config/config.go b/config/config.go index b65c0bdee..be46ad544 100644 --- a/config/config.go +++ b/config/config.go @@ -15,6 +15,7 @@ package config import ( + "context" "crypto/md5" "encoding/hex" "fmt" @@ -32,6 +33,15 @@ import ( "github.com/spf13/viper" "github.com/zhenghaoz/gorse/base/log" "github.com/zhenghaoz/gorse/storage" + "go.opentelemetry.io/otel/exporters/jaeger" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/exporters/zipkin" + "go.opentelemetry.io/otel/sdk/resource" + tracesdk "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.8.0" + "go.opentelemetry.io/otel/trace" "go.uber.org/zap" ) @@ -47,6 +57,7 @@ type Config struct { Master MasterConfig `mapstructure:"master"` Server ServerConfig `mapstructure:"server"` Recommend RecommendConfig `mapstructure:"recommend"` + Tracing TracingConfig `mapstructure:"tracing"` } // DatabaseConfig is the configuration for the database. @@ -148,6 +159,14 @@ type OnlineConfig struct { NumFeedbackFallbackItemBased int `mapstructure:"num_feedback_fallback_item_based" validate:"gt=0"` } +type TracingConfig struct { + EnableTracing bool `mapstructure:"enable_tracing"` + Exporter string `mapstructure:"exporter" validate:"oneof=jaeger zipkin otlp otlphttp"` + CollectorEndpoint string `mapstructure:"collector_endpoint"` + Sampler string `mapstructure:"sampler"` + Ratio float64 `mapstructure:"ratio"` +} + func GetDefaultConfig() *Config { return &Config{ Master: MasterConfig{ @@ -214,6 +233,10 @@ func GetDefaultConfig() *Config { NumFeedbackFallbackItemBased: 10, }, }, + Tracing: TracingConfig{ + Exporter: "jaeger", + Sampler: "always", + }, } } @@ -359,6 +382,73 @@ func (config *OfflineConfig) GetExploreRecommend(key string) (value float64, exi return } +func (config *TracingConfig) NewTracerProvider() (trace.TracerProvider, error) { + if !config.EnableTracing { + return trace.NewNoopTracerProvider(), nil + } + + var exporter tracesdk.SpanExporter + var err error + switch config.Exporter { + case "jaeger": + exporter, err = jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(config.CollectorEndpoint))) + if err != nil { + return nil, errors.Trace(err) + } + case "zipkin": + exporter, err = zipkin.New(config.CollectorEndpoint) + if err != nil { + return nil, errors.Trace(err) + } + case "otlp": + client := otlptracegrpc.NewClient(otlptracegrpc.WithInsecure(), otlptracegrpc.WithEndpoint(config.CollectorEndpoint)) + exporter, err = otlptrace.New(context.TODO(), client) + if err != nil { + return nil, errors.Trace(err) + } + case "otlphttp": + client := otlptracehttp.NewClient(otlptracehttp.WithInsecure(), otlptracehttp.WithEndpoint(config.CollectorEndpoint)) + exporter, err = otlptrace.New(context.TODO(), client) + if err != nil { + return nil, errors.Trace(err) + } + default: + return nil, errors.NotSupportedf("exporter %s", config.Exporter) + } + + var sampler tracesdk.Sampler + switch config.Sampler { + case "always": + sampler = tracesdk.AlwaysSample() + case "never": + sampler = tracesdk.NeverSample() + case "ratio": + sampler = tracesdk.TraceIDRatioBased(config.Ratio) + default: + return nil, errors.NotSupportedf("sampler %s", config.Sampler) + } + + return tracesdk.NewTracerProvider( + tracesdk.WithSampler(sampler), + tracesdk.WithBatcher(exporter), + tracesdk.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String("gorse"), + )), + ), nil +} + +func (config *TracingConfig) Equal(other TracingConfig) bool { + if config == nil { + return false + } + return config.EnableTracing == other.EnableTracing && + config.Exporter == other.Exporter && + config.CollectorEndpoint == other.CollectorEndpoint && + config.Sampler == other.Sampler && + config.Ratio == other.Ratio +} + func setDefault() { defaultConfig := GetDefaultConfig() // [master] @@ -416,6 +506,9 @@ func setDefault() { // [recommend.online] viper.SetDefault("recommend.online.fallback_recommend", defaultConfig.Recommend.Online.FallbackRecommend) viper.SetDefault("recommend.online.num_feedback_fallback_item_based", defaultConfig.Recommend.Online.NumFeedbackFallbackItemBased) + // [tracing] + viper.SetDefault("tracing.exporter", defaultConfig.Tracing.Exporter) + viper.SetDefault("tracing.sampler", defaultConfig.Tracing.Sampler) } type configBinding struct { diff --git a/config/config.toml.template b/config/config.toml.template index 136031799..689db3f54 100644 --- a/config/config.toml.template +++ b/config/config.toml.template @@ -227,3 +227,20 @@ fallback_recommend = ["item_based", "latest"] # The number of feedback used in fallback item-based similar recommendation. The default values is 10. num_feedback_fallback_item_based = 10 + +[tracing] + +# Enable tracing for REST APIs. The default value is false. +enable_tracing = false + +# The type of tracing exporters should be one of "jaeger", "zipkin", "otlp" and "otlphttp". The default value is "jaeger". +exporter = "jaeger" + +# The endpoint of tracing collector. +collector_endpoint = "http://localhost:14268/api/traces" + +# The type of tracing sampler should be one of "always", "never" and "ratio". The default value is "always". +sampler = "always" + +# The ratio of ratio based sampler. The default value is 1. +ratio = 1 diff --git a/config/config_test.go b/config/config_test.go index a507ec853..40592f708 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -119,6 +119,12 @@ func TestUnmarshal(t *testing.T) { // [recommend.online] assert.Equal(t, []string{"item_based", "latest"}, config.Recommend.Online.FallbackRecommend) assert.Equal(t, 10, config.Recommend.Online.NumFeedbackFallbackItemBased) + // [tracing] + assert.False(t, config.Tracing.EnableTracing) + assert.Equal(t, "jaeger", config.Tracing.Exporter) + assert.Equal(t, "http://localhost:14268/api/traces", config.Tracing.CollectorEndpoint) + assert.Equal(t, "always", config.Tracing.Sampler) + assert.Equal(t, 1.0, config.Tracing.Ratio) } func TestSetDefault(t *testing.T) { diff --git a/go.mod b/go.mod index 47cd9fb70..dd250521f 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.18 require ( github.com/ReneKroon/ttlcache/v2 v2.11.0 + github.com/XSAM/otelsql v0.17.0 github.com/alicebob/miniredis/v2 v2.23.0 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de github.com/benhoyt/goawk v1.20.0 @@ -15,7 +16,8 @@ require ( github.com/go-playground/locales v0.14.0 github.com/go-playground/universal-translator v0.18.0 github.com/go-playground/validator/v10 v10.11.0 - github.com/go-redis/redis/v9 v9.0.0-beta.2 + github.com/go-redis/redis/extra/redisotel/v9 v9.0.0-rc.1 + github.com/go-redis/redis/v9 v9.0.0-rc.1 github.com/go-resty/resty/v2 v2.7.0 github.com/go-sql-driver/mysql v1.6.0 github.com/google/uuid v1.3.0 @@ -42,11 +44,22 @@ require ( github.com/steinfletcher/apitest v1.5.12 github.com/stretchr/testify v1.8.0 github.com/thoas/go-funk v0.9.2 - go.mongodb.org/mongo-driver v1.10.1 + go.mongodb.org/mongo-driver v1.10.3 + go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.36.4 + go.opentelemetry.io/contrib/instrumentation/go.mongodb.org/mongo-driver/mongo/otelmongo v0.36.4 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.36.4 + go.opentelemetry.io/otel v1.11.1 + go.opentelemetry.io/otel/exporters/jaeger v1.11.1 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.1 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.1 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.1 + go.opentelemetry.io/otel/exporters/zipkin v1.11.1 + go.opentelemetry.io/otel/sdk v1.11.1 + go.opentelemetry.io/otel/trace v1.11.1 go.uber.org/atomic v1.10.0 go.uber.org/zap v1.22.0 golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e - google.golang.org/grpc v1.48.0 + google.golang.org/grpc v1.50.1 google.golang.org/protobuf v1.28.1 gopkg.in/yaml.v2 v2.4.0 gorm.io/driver/clickhouse v0.4.2 @@ -64,16 +77,22 @@ require ( require ( github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect + github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/spec v0.20.6 // indirect github.com/go-openapi/swag v0.21.1 // indirect + github.com/go-redis/redis/extra/rediscmd/v9 v9.0.0-rc.1 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect @@ -89,7 +108,7 @@ require ( github.com/jinzhu/now v1.1.5 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/klauspost/compress v1.15.8 // indirect + github.com/klauspost/compress v1.15.11 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -102,6 +121,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/montanaflynn/stats v0.6.6 // indirect + github.com/openzipkin/zipkin-go v0.4.1 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.2 // indirect github.com/pkg/errors v0.9.1 // indirect @@ -122,16 +142,19 @@ require ( github.com/xdg-go/stringprep v1.0.3 // indirect github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1 // indirect + go.opentelemetry.io/otel/metric v0.33.0 // indirect + go.opentelemetry.io/proto/otlp v0.19.0 // indirect go.uber.org/multierr v1.8.0 // indirect - golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect + golang.org/x/crypto v0.0.0-20221010152910-d6f0a8c073c2 // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/net v0.0.0-20220708220712-1185a9018129 // indirect - golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect - golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect + golang.org/x/net v0.0.0-20221004154528-8021a29435af // indirect + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect + golang.org/x/sys v0.0.0-20221010170243-090e33056c14 // indirect golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 // indirect - golang.org/x/text v0.3.7 // indirect - golang.org/x/tools v0.1.11 // indirect - google.golang.org/genproto v0.0.0-20220719170305-83ca9fad585f // indirect + golang.org/x/text v0.3.8 // indirect + golang.org/x/tools v0.1.12 // indirect + google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e // indirect gopkg.in/ini.v1 v1.66.6 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/uint128 v1.2.0 // indirect diff --git a/go.sum b/go.sum index 3a7153297..66d2365da 100644 --- a/go.sum +++ b/go.sum @@ -40,10 +40,13 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/ReneKroon/ttlcache/v2 v2.11.0 h1:OvlcYFYi941SBN3v9dsDcC2N8vRxyHcCmJb3Vl4QMoM= github.com/ReneKroon/ttlcache/v2 v2.11.0/go.mod h1:mBxvsNY+BT8qLLd6CuAJubbKo6r0jh3nb5et22bbfGY= +github.com/XSAM/otelsql v0.17.0 h1:eq9yWwFY6t+nruMmjzhbdbkIo+HnUTziimkBbIqNKwM= +github.com/XSAM/otelsql v0.17.0/go.mod h1:zGN8fF5r7Xclr92eMl8JM8pYquSWV8Kjf2fUeWvpd3Y= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -66,7 +69,10 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.2.1 h1:M+/hrU9xlMp7t4TyTDQW97d3tRPVuKFC6zBEK16QnXY= github.com/bits-and-blooms/bitset v1.2.1/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= +github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -80,8 +86,9 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= @@ -107,11 +114,16 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/set v0.2.1 h1:nn2CaJyknWE/6txyUDGwysr3G5QC6xWB/PtVjPBbeaA= github.com/fatih/set v0.2.1/go.mod h1:+RKtMCH+favT2+3YecHGxcc0b4KyVWA1QWWJUs4E0CI= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -126,6 +138,11 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -147,17 +164,24 @@ github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/j github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/validator/v10 v10.11.0 h1:0W+xRM511GY47Yy3bZUbJVitCNg2BOGlCyvTqsp/xIw= github.com/go-playground/validator/v10 v10.11.0/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= -github.com/go-redis/redis/v9 v9.0.0-beta.2 h1:ZSr84TsnQyKMAg8gnV+oawuQezeJR11/09THcWCQzr4= -github.com/go-redis/redis/v9 v9.0.0-beta.2/go.mod h1:Bldcd/M/bm9HbnNPi/LUtYBSD8ttcZYBMupwMXhdU0o= +github.com/go-redis/redis/extra/rediscmd/v9 v9.0.0-rc.1 h1:g7nnZITVk5EKGu4Z34PDTfy6ZP44Xl53HNUYsvtKS5Y= +github.com/go-redis/redis/extra/rediscmd/v9 v9.0.0-rc.1/go.mod h1:3RRYklqyVeBMgYMdsLA7sx7l1txAiA67DvVMw0NppKQ= +github.com/go-redis/redis/extra/redisotel/v9 v9.0.0-rc.1 h1:p1AdAQMiSFLgFX0/irDucHBzBjRmH2VDu0GdTtKXTIw= +github.com/go-redis/redis/extra/redisotel/v9 v9.0.0-rc.1/go.mod h1:cLtoNcdkUJWquixbfnxgYB+Y831Ec1nFLPL736AUMa4= +github.com/go-redis/redis/v9 v9.0.0-rc.1 h1:/+bS+yeUnanqAbuD3QwlejzQZ+4eqgfUtFTG4b+QnXs= +github.com/go-redis/redis/v9 v9.0.0-rc.1/go.mod h1:8et+z03j0l8N+DvsVnclzjf3Dl/pFHgRk+2Ct1qw66A= github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -202,7 +226,8 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -217,6 +242,7 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -236,6 +262,8 @@ github.com/gorse-io/oracle v1.2.2-0.20220717033501-725373bd8ae8/go.mod h1:Sjtk+M github.com/gorse-io/sqlite v1.3.3-0.20220713123255-c322aec4e59e h1:uPQtYQzG1QcC3Qbv+tuEe8Q2l++V4KEcqYSSwB9qobg= github.com/gorse-io/sqlite v1.3.3-0.20220713123255-c322aec4e59e/go.mod h1:PmIOwYnI+F1lRKd6F/PdLXGgI8GZ5H8x8z1yx0+0bmQ= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/hashicorp/go-version v1.5.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= @@ -245,6 +273,7 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/haxii/go-swagger-ui v3.19.4+incompatible h1:SBwmVIj/dlz+gBwC0X9Ho61nqEzW0gHigjsmuFZ4Z7E= github.com/haxii/go-swagger-ui v3.19.4+incompatible/go.mod h1:vHbZPVE2aQaPb571vtqWKTT0jFfthdaNy8IaFbof4oQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -324,8 +353,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.8 h1:JahtItbkWjf2jzm/T+qgMxkP9EMHsqEUA6vCMGmXvhA= -github.com/klauspost/compress v1.15.8/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= +github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/cpuid/v2 v2.1.0 h1:eyi1Ad2aNJMW95zcSbmGg7Cg6cq3ADwLpMAP96d8rF0= github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -395,9 +424,26 @@ github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/gomega v1.20.0 h1:8W0cWlwFkflGPLltQvLRB7ZVD5HuP6ng320w2IS245Q= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= +github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.21.1 h1:OB/euWYIExnPBohllTicTHmGTrMaqJ67nIu80j0/uEM= +github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= +github.com/openzipkin/zipkin-go v0.4.1 h1:kNd/ST2yLLWhaWrkgchya40TJabe8Hioj9udfPcEO5A= +github.com/openzipkin/zipkin-go v0.4.1/go.mod h1:qY0VqDSN1pOBN94dBc6w2GJlWLiovAyg7Qt6/I9HecM= github.com/orcaman/concurrent-map v1.0.0 h1:I/2A2XPCb4IuQWcQhBhSwGfiuybl/J0ev9HDbW65HOY= github.com/orcaman/concurrent-map v1.0.0/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= @@ -449,8 +495,8 @@ github.com/rivo/uniseg v0.3.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= @@ -473,6 +519,7 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.9.2 h1:j49Hj62F0n+DaZ1dDCvhABaPNSGNkt32oRFxI33IEMw= github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= @@ -490,6 +537,7 @@ github.com/steinfletcher/apitest v1.5.12/go.mod h1:cf7Bneo52IIAgpqhP8xaLlzWgAiQ9 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -521,19 +569,56 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v0.0.0-20210529063254-f4c35e4016d9/go.mod h1:E1AXubJBdNmFERAOucpDIxNzeGfLzg0mYh+UfMWdChA= github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64 h1:5mLPGnFdSsevFRFc9q3yYbBkB6tsm4aCwwQV/j1JQAQ= github.com/yuin/gopher-lua v0.0.0-20220504180219-658193537a64/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= -go.mongodb.org/mongo-driver v1.10.1 h1:NujsPveKwHaWuKUer/ceo9DzEe7HIj1SlJ6uvXZG0S4= -go.mongodb.org/mongo-driver v1.10.1/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8= +go.mongodb.org/mongo-driver v1.10.3 h1:XDQEvmh6z1EUsXuIkXE9TaVeqHw6SwS1uf93jFs0HBA= +go.mongodb.org/mongo-driver v1.10.3/go.mod h1:z4XpeoU6w+9Vht+jAFyLgVrD+jGSQQe0+CBWFHNiHt8= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.36.4 h1:9nyzSapoM0ZYuZsHjZColmVm6tCKJo1boLQ7NEDReo8= +go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful v0.36.4/go.mod h1:FW//LC5AQiDTWo1Jz9So0UMlsm2xT/vJf8UiHytP0FI= +go.opentelemetry.io/contrib/instrumentation/go.mongodb.org/mongo-driver/mongo/otelmongo v0.36.4 h1:IKvVGMy0s5MH0cKfwmwiHVtnrVOFuHU/wznLa8eN+Cs= +go.opentelemetry.io/contrib/instrumentation/go.mongodb.org/mongo-driver/mongo/otelmongo v0.36.4/go.mod h1:mHrZBcL5tUSxYX1emmDCNDDf9an1PedCEGum4p9+Ep8= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.36.4 h1:aUEBEdCa6iamGzg6fuYxDA8ThxvOG240mAvWDU+XLio= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.36.4/go.mod h1:l2MdsbKTocpPS5nQZscqTR9jd8u96VYZdcpF8Sye7mA= +go.opentelemetry.io/contrib/propagators/b3 v1.11.1 h1:icQ6ttRV+r/2fnU46BIo/g/mPu6Rs5Ug8Rtohe3KqzI= +go.opentelemetry.io/otel v1.9.0/go.mod h1:np4EoPGzoPs3O67xUVNoPPcmSvsfOxNlNA4F4AC+0Eo= +go.opentelemetry.io/otel v1.10.0/go.mod h1:NbvWjCthWHKBEUMpf0/v8ZRZlni86PpGFEMA9pnQSnQ= +go.opentelemetry.io/otel v1.11.1 h1:4WLLAmcfkmDk2ukNXJyq3/kiz/3UzCaYq6PskJsaou4= +go.opentelemetry.io/otel v1.11.1/go.mod h1:1nNhXBbWSD0nsL38H6btgnFN2k4i0sNLHNNMZMSbUGE= +go.opentelemetry.io/otel/exporters/jaeger v1.11.1 h1:F9Io8lqWdGyIbY3/SOGki34LX/l+7OL0gXNxjqwcbuQ= +go.opentelemetry.io/otel/exporters/jaeger v1.11.1/go.mod h1:lRa2w3bQ4R4QN6zYsDgy7tEezgoKEu7Ow2g35Y75+KI= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1 h1:X2GndnMCsUPh6CiY2a+frAbNsXaPLbB0soHRYhAZ5Ig= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.1/go.mod h1:i8vjiSzbiUC7wOQplijSXMYUpNM93DtlS5CbUT+C6oQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.1 h1:MEQNafcNCB0uQIti/oHgU7CZpUMYQ7qigBwMVKycHvc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.1/go.mod h1:19O5I2U5iys38SsmT2uDJja/300woyzE1KPIQxEUBUc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.1 h1:LYyG/f1W/jzAix16jbksJfMQFpOH/Ma6T639pVPMgfI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.1/go.mod h1:QrRRQiY3kzAoYPNLP0W/Ikg0gR6V3LMc+ODSxr7yyvg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.1 h1:tFl63cpAAcD9TOU6U8kZU7KyXuSRYAZlbx1C61aaB74= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.1/go.mod h1:X620Jww3RajCJXw/unA+8IRTgxkdS7pi+ZwK9b7KUJk= +go.opentelemetry.io/otel/exporters/zipkin v1.11.1 h1:JlJ3/oQoyqlrPDCfsSVFcHgGeHvZq+hr1VPWtiYCXTo= +go.opentelemetry.io/otel/exporters/zipkin v1.11.1/go.mod h1:T4S6aVwIS1+MHA+dJHCcPROtZe6ORwnv5vMKPRapsFw= +go.opentelemetry.io/otel/metric v0.32.1/go.mod h1:iLPP7FaKMAD5BIxJ2VX7f2KTuz//0QK2hEUyti5psqQ= +go.opentelemetry.io/otel/metric v0.33.0 h1:xQAyl7uGEYvrLAiV/09iTJlp1pZnQ9Wl793qbVvED1E= +go.opentelemetry.io/otel/metric v0.33.0/go.mod h1:QlTYc+EnYNq/M2mNk1qDDMRLpqCOj2f/r5c7Fd5FYaI= +go.opentelemetry.io/otel/sdk v1.9.0/go.mod h1:AEZc8nt5bd2F7BC24J5R0mrjYnpEgYHyTcM/vrSple4= +go.opentelemetry.io/otel/sdk v1.11.1 h1:F7KmQgoHljhUuJyA+9BiU+EkJfyX5nVVF4wyzWZpKxs= +go.opentelemetry.io/otel/sdk v1.11.1/go.mod h1:/l3FE4SupHJ12TduVjUkZtlfFqDCQJlOlithYrdktys= +go.opentelemetry.io/otel/trace v1.9.0/go.mod h1:2737Q0MuG8q1uILYm2YYVkAyLtOofiTNGg6VODnOiPo= +go.opentelemetry.io/otel/trace v1.10.0/go.mod h1:Sij3YYczqAdz+EhmGhE6TpTxUO5/F/AzrK+kxfGqySM= +go.opentelemetry.io/otel/trace v1.11.1 h1:ofxdnzsNrGBYXbP7t7zpUK281+go5rF7dvdIZXF8gdQ= +go.opentelemetry.io/otel/trace v1.11.1/go.mod h1:f/Q9G7vzk5u91PhbmKbg1Qn0rzH1LJ4vbPHFGkTPtOk= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= +go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -542,8 +627,8 @@ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -576,8 +661,8 @@ golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220131195533-30dcbda58838/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa h1:zuSxTR4o9y82ebqCUJYNGJbGPo6sKVl54f/TVDObg1c= -golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20221010152910-d6f0a8c073c2 h1:x8vtB3zMecnlqZIwJNUUpwYKYSqCz5jXbiyv0ZJJZeI= +golang.org/x/crypto v0.0.0-20221010152910-d6f0a8c073c2/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -614,10 +699,12 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -641,6 +728,7 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -652,13 +740,17 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220708220712-1185a9018129 h1:vucSRfWwTsoXro7P+3Cjlr6flUMtzCwzlvkxEQtHHB0= -golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20221004154528-8021a29435af h1:wv66FM3rLZGPdxpYL+ApnDe2HzHcTFta3z5nsc13wI4= +golang.org/x/net v0.0.0-20221004154528-8021a29435af/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -669,6 +761,7 @@ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -681,10 +774,11 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -699,8 +793,11 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -725,6 +822,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -738,13 +836,18 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14 h1:k5II8e6QD8mITdi+okbbmR/cIyEbeXLBhy5Ha4nevyc= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -756,9 +859,11 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -813,13 +918,15 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.11 h1:loJ25fNOEhSXfHrpoGj91eCUThwdNX6u24rO1xnNteY= -golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -889,8 +996,9 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20220719170305-83ca9fad585f h1:P8EiVSxZwC6xH2niv2N66aqwMtYFg+D54gbjpcqKJtM= -google.golang.org/genproto v0.0.0-20220719170305-83ca9fad585f/go.mod h1:GkXuJDJ6aQ7lnJcRF+SJVgFdQhypqgl3LB1C9vabdRE= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e h1:halCgTFuLWDRD61piiNSxPsARANGD3Xl16hPrLgLiIg= +google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqwhZAwq4wsRUaVG555sVgsNmIjRtO7t/JH29U= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -909,8 +1017,10 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.48.0 h1:rQOsyJ/8+ufEDJd/Gdsz7HG220Mh9HAhFHRGnIjda0w= -google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.50.1 h1:DS/BukOZWp8s6p4Dt/tOaJaTQyPyOoCcrjroHuCeLzY= +google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -924,6 +1034,7 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -934,10 +1045,12 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/ini.v1 v1.66.6 h1:LATuAqN/shcYAOkv3wl2L4rkaKqkcgTBQjOyYDvcPKI= gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/master/master.go b/master/master.go index d8be57afc..5c549cd2e 100644 --- a/master/master.go +++ b/master/master.go @@ -39,6 +39,8 @@ import ( "github.com/zhenghaoz/gorse/server" "github.com/zhenghaoz/gorse/storage/cache" "github.com/zhenghaoz/gorse/storage/data" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" "go.uber.org/zap" "google.golang.org/grpc" ) @@ -101,6 +103,14 @@ type Master struct { // NewMaster creates a master node. func NewMaster(cfg *config.Config, cacheFile string, managedMode bool) *Master { rand.Seed(time.Now().UnixNano()) + // setup trace provider + tp, err := cfg.Tracing.NewTracerProvider() + if err != nil { + log.Logger().Fatal("failed to create trace provider", zap.Error(err)) + } + otel.SetTracerProvider(tp) + otel.SetErrorHandler(log.GetErrorHandler()) + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})) // create task monitor taskMonitor := task.NewTaskMonitor() for _, taskName := range []string{TaskLoadDataset, TaskFindItemNeighbors, TaskFindUserNeighbors, @@ -439,7 +449,8 @@ func (m *Master) RunManagedTasksLoop() { } func (m *Master) checkDataImported() bool { - isDataImported, err := m.CacheClient.Get(cache.Key(cache.GlobalMeta, cache.DataImported)).Integer() + ctx := context.Background() + isDataImported, err := m.CacheClient.Get(ctx, cache.Key(cache.GlobalMeta, cache.DataImported)).Integer() if err != nil { if !errors.Is(err, errors.NotFound) { log.Logger().Error("failed to read meta", zap.Error(err)) @@ -447,7 +458,7 @@ func (m *Master) checkDataImported() bool { return false } if isDataImported > 0 { - err = m.CacheClient.Set(cache.Integer(cache.Key(cache.GlobalMeta, cache.DataImported), 0)) + err = m.CacheClient.Set(ctx, cache.Integer(cache.Key(cache.GlobalMeta, cache.DataImported), 0)) if err != nil { log.Logger().Error("failed to write meta", zap.Error(err)) } @@ -457,7 +468,8 @@ func (m *Master) checkDataImported() bool { } func (m *Master) notifyDataImported() { - err := m.CacheClient.Set(cache.Integer(cache.Key(cache.GlobalMeta, cache.DataImported), 1)) + ctx := context.Background() + err := m.CacheClient.Set(ctx, cache.Integer(cache.Key(cache.GlobalMeta, cache.DataImported), 1)) if err != nil { log.Logger().Error("failed to write meta", zap.Error(err)) } diff --git a/master/rest.go b/master/rest.go index ef9204a3e..31cf2fcb8 100644 --- a/master/rest.go +++ b/master/rest.go @@ -16,6 +16,7 @@ package master import ( "bufio" + "context" "encoding/json" "fmt" "io" @@ -380,8 +381,12 @@ func (m *Master) checkLogin(request *http.Request) bool { return false } -func (m *Master) getCategories(_ *restful.Request, response *restful.Response) { - categories, err := m.CacheClient.GetSet(cache.ItemCategories) +func (m *Master) getCategories(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } + categories, err := m.CacheClient.GetSet(ctx, cache.ItemCategories) if err != nil { server.InternalServerError(response, err) return @@ -462,35 +467,39 @@ type Status struct { MatchingIndexRecall float32 } -func (m *Master) getStats(_ *restful.Request, response *restful.Response) { +func (m *Master) getStats(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } status := Status{BinaryVersion: version.Version} var err error // read number of users - if status.NumUsers, err = m.CacheClient.Get(cache.Key(cache.GlobalMeta, cache.NumUsers)).Integer(); err != nil { + if status.NumUsers, err = m.CacheClient.Get(ctx, cache.Key(cache.GlobalMeta, cache.NumUsers)).Integer(); err != nil { log.ResponseLogger(response).Warn("failed to get number of users", zap.Error(err)) } // read number of items - if status.NumItems, err = m.CacheClient.Get(cache.Key(cache.GlobalMeta, cache.NumItems)).Integer(); err != nil { + if status.NumItems, err = m.CacheClient.Get(ctx, cache.Key(cache.GlobalMeta, cache.NumItems)).Integer(); err != nil { log.ResponseLogger(response).Warn("failed to get number of items", zap.Error(err)) } // read number of user labels - if status.NumUserLabels, err = m.CacheClient.Get(cache.Key(cache.GlobalMeta, cache.NumUserLabels)).Integer(); err != nil { + if status.NumUserLabels, err = m.CacheClient.Get(ctx, cache.Key(cache.GlobalMeta, cache.NumUserLabels)).Integer(); err != nil { log.ResponseLogger(response).Warn("failed to get number of user labels", zap.Error(err)) } // read number of item labels - if status.NumItemLabels, err = m.CacheClient.Get(cache.Key(cache.GlobalMeta, cache.NumItemLabels)).Integer(); err != nil { + if status.NumItemLabels, err = m.CacheClient.Get(ctx, cache.Key(cache.GlobalMeta, cache.NumItemLabels)).Integer(); err != nil { log.ResponseLogger(response).Warn("failed to get number of item labels", zap.Error(err)) } // read number of total positive feedback - if status.NumTotalPosFeedback, err = m.CacheClient.Get(cache.Key(cache.GlobalMeta, cache.NumTotalPosFeedbacks)).Integer(); err != nil { + if status.NumTotalPosFeedback, err = m.CacheClient.Get(ctx, cache.Key(cache.GlobalMeta, cache.NumTotalPosFeedbacks)).Integer(); err != nil { log.ResponseLogger(response).Warn("failed to get number of total positive feedbacks", zap.Error(err)) } // read number of valid positive feedback - if status.NumValidPosFeedback, err = m.CacheClient.Get(cache.Key(cache.GlobalMeta, cache.NumValidPosFeedbacks)).Integer(); err != nil { + if status.NumValidPosFeedback, err = m.CacheClient.Get(ctx, cache.Key(cache.GlobalMeta, cache.NumValidPosFeedbacks)).Integer(); err != nil { log.ResponseLogger(response).Warn("failed to get number of valid positive feedbacks", zap.Error(err)) } // read number of valid negative feedback - if status.NumValidNegFeedback, err = m.CacheClient.Get(cache.Key(cache.GlobalMeta, cache.NumValidNegFeedbacks)).Integer(); err != nil { + if status.NumValidNegFeedback, err = m.CacheClient.Get(ctx, cache.Key(cache.GlobalMeta, cache.NumValidNegFeedbacks)).Integer(); err != nil { log.ResponseLogger(response).Warn("failed to get number of valid negative feedbacks", zap.Error(err)) } // count the number of workers and servers @@ -505,27 +514,27 @@ func (m *Master) getStats(_ *restful.Request, response *restful.Response) { } m.nodesInfoMutex.Unlock() // read popular items update time - if status.PopularItemsUpdateTime, err = m.CacheClient.Get(cache.Key(cache.GlobalMeta, cache.LastUpdatePopularItemsTime)).Time(); err != nil { + if status.PopularItemsUpdateTime, err = m.CacheClient.Get(ctx, cache.Key(cache.GlobalMeta, cache.LastUpdatePopularItemsTime)).Time(); err != nil { log.ResponseLogger(response).Warn("failed to get popular items update time", zap.Error(err)) } // read the latest items update time - if status.LatestItemsUpdateTime, err = m.CacheClient.Get(cache.Key(cache.GlobalMeta, cache.LastUpdateLatestItemsTime)).Time(); err != nil { + if status.LatestItemsUpdateTime, err = m.CacheClient.Get(ctx, cache.Key(cache.GlobalMeta, cache.LastUpdateLatestItemsTime)).Time(); err != nil { log.ResponseLogger(response).Warn("failed to get latest items update time", zap.Error(err)) } status.MatchingModelScore = m.rankingScore status.RankingModelScore = m.clickScore // read last fit matching model time - if status.MatchingModelFitTime, err = m.CacheClient.Get(cache.Key(cache.GlobalMeta, cache.LastFitMatchingModelTime)).Time(); err != nil { + if status.MatchingModelFitTime, err = m.CacheClient.Get(ctx, cache.Key(cache.GlobalMeta, cache.LastFitMatchingModelTime)).Time(); err != nil { log.ResponseLogger(response).Warn("failed to get last fit matching model time", zap.Error(err)) } // read last fit ranking model time - if status.RankingModelFitTime, err = m.CacheClient.Get(cache.Key(cache.GlobalMeta, cache.LastFitRankingModelTime)).Time(); err != nil { + if status.RankingModelFitTime, err = m.CacheClient.Get(ctx, cache.Key(cache.GlobalMeta, cache.LastFitRankingModelTime)).Time(); err != nil { log.ResponseLogger(response).Warn("failed to get last fit ranking model time", zap.Error(err)) } // read user neighbor index recall var temp string if m.Config.Recommend.UserNeighbors.EnableIndex { - if temp, err = m.CacheClient.Get(cache.Key(cache.GlobalMeta, cache.UserNeighborIndexRecall)).String(); err != nil { + if temp, err = m.CacheClient.Get(ctx, cache.Key(cache.GlobalMeta, cache.UserNeighborIndexRecall)).String(); err != nil { log.ResponseLogger(response).Warn("failed to get user neighbor index recall", zap.Error(err)) } else { status.UserNeighborIndexRecall = encoding.ParseFloat32(temp) @@ -533,7 +542,7 @@ func (m *Master) getStats(_ *restful.Request, response *restful.Response) { } // read item neighbor index recall if m.Config.Recommend.ItemNeighbors.EnableIndex { - if temp, err = m.CacheClient.Get(cache.Key(cache.GlobalMeta, cache.ItemNeighborIndexRecall)).String(); err != nil { + if temp, err = m.CacheClient.Get(ctx, cache.Key(cache.GlobalMeta, cache.ItemNeighborIndexRecall)).String(); err != nil { log.ResponseLogger(response).Warn("failed to get item neighbor index recall", zap.Error(err)) } else { status.ItemNeighborIndexRecall = encoding.ParseFloat32(temp) @@ -541,7 +550,7 @@ func (m *Master) getStats(_ *restful.Request, response *restful.Response) { } // read matching index recall if m.Config.Recommend.Collaborative.EnableIndex { - if temp, err = m.CacheClient.Get(cache.Key(cache.GlobalMeta, cache.MatchingIndexRecall)).String(); err != nil { + if temp, err = m.CacheClient.Get(ctx, cache.Key(cache.GlobalMeta, cache.MatchingIndexRecall)).String(); err != nil { log.ResponseLogger(response).Warn("failed to get matching index recall", zap.Error(err)) } else { status.MatchingIndexRecall = encoding.ParseFloat32(temp) @@ -566,6 +575,10 @@ func (m *Master) getTasks(_ *restful.Request, response *restful.Response) { } func (m *Master) getRates(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } // Parse parameters n, err := server.ParseInt(request, "n", 100) if err != nil { @@ -574,7 +587,7 @@ func (m *Master) getRates(request *restful.Request, response *restful.Response) } measurements := make(map[string][]server.Measurement, len(m.Config.Recommend.DataSource.PositiveFeedbackTypes)) for _, feedbackType := range m.Config.Recommend.DataSource.PositiveFeedbackTypes { - measurements[feedbackType], err = m.RestServer.GetMeasurements(cache.Key(PositiveFeedbackRate, feedbackType), n) + measurements[feedbackType], err = m.RestServer.GetMeasurements(ctx, cache.Key(PositiveFeedbackRate, feedbackType), n) if err != nil { server.InternalServerError(response, err) return @@ -595,10 +608,14 @@ type User struct { } func (m *Master) getUser(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } // get user id userId := request.PathParameter("user-id") // get user - user, err := m.DataClient.GetUser(userId) + user, err := m.DataClient.GetUser(ctx, userId) if err != nil { if errors.Is(err, errors.NotFound) { server.PageNotFound(response, err) @@ -608,11 +625,11 @@ func (m *Master) getUser(request *restful.Request, response *restful.Response) { return } detail := User{User: user} - if detail.LastActiveTime, err = m.CacheClient.Get(cache.Key(cache.LastModifyUserTime, user.UserId)).Time(); err != nil && !errors.Is(err, errors.NotFound) { + if detail.LastActiveTime, err = m.CacheClient.Get(ctx, cache.Key(cache.LastModifyUserTime, user.UserId)).Time(); err != nil && !errors.Is(err, errors.NotFound) { server.InternalServerError(response, err) return } - if detail.LastUpdateTime, err = m.CacheClient.Get(cache.Key(cache.LastUpdateUserRecommendTime, user.UserId)).Time(); err != nil && !errors.Is(err, errors.NotFound) { + if detail.LastUpdateTime, err = m.CacheClient.Get(ctx, cache.Key(cache.LastUpdateUserRecommendTime, user.UserId)).Time(); err != nil && !errors.Is(err, errors.NotFound) { server.InternalServerError(response, err) return } @@ -620,6 +637,10 @@ func (m *Master) getUser(request *restful.Request, response *restful.Response) { } func (m *Master) getUsers(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } // Authorize cursor := request.QueryParameter("cursor") n, err := server.ParseInt(request, "n", m.Config.Server.DefaultN) @@ -628,7 +649,7 @@ func (m *Master) getUsers(request *restful.Request, response *restful.Response) return } // get all users - cursor, users, err := m.DataClient.GetUsers(cursor, n) + cursor, users, err := m.DataClient.GetUsers(ctx, cursor, n) if err != nil { server.InternalServerError(response, err) return @@ -636,11 +657,11 @@ func (m *Master) getUsers(request *restful.Request, response *restful.Response) details := make([]User, len(users)) for i, user := range users { details[i].User = user - if details[i].LastActiveTime, err = m.CacheClient.Get(cache.Key(cache.LastModifyUserTime, user.UserId)).Time(); err != nil && !errors.Is(err, errors.NotFound) { + if details[i].LastActiveTime, err = m.CacheClient.Get(ctx, cache.Key(cache.LastModifyUserTime, user.UserId)).Time(); err != nil && !errors.Is(err, errors.NotFound) { server.InternalServerError(response, err) return } - if details[i].LastUpdateTime, err = m.CacheClient.Get(cache.Key(cache.LastUpdateUserRecommendTime, user.UserId)).Time(); err != nil && !errors.Is(err, errors.NotFound) { + if details[i].LastUpdateTime, err = m.CacheClient.Get(ctx, cache.Key(cache.LastUpdateUserRecommendTime, user.UserId)).Time(); err != nil && !errors.Is(err, errors.NotFound) { server.InternalServerError(response, err) return } @@ -649,6 +670,10 @@ func (m *Master) getUsers(request *restful.Request, response *restful.Response) } func (m *Master) getRecommend(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } // parse arguments recommender := request.PathParameter("recommender") userId := request.PathParameter("user-id") @@ -661,13 +686,13 @@ func (m *Master) getRecommend(request *restful.Request, response *restful.Respon var results []string switch recommender { case "offline": - results, err = m.Recommend(response, userId, category, n, m.RecommendOffline) + results, err = m.Recommend(ctx, response, userId, category, n, m.RecommendOffline) case "collaborative": - results, err = m.Recommend(response, userId, category, n, m.RecommendCollaborative) + results, err = m.Recommend(ctx, response, userId, category, n, m.RecommendCollaborative) case "user_based": - results, err = m.Recommend(response, userId, category, n, m.RecommendUserBased) + results, err = m.Recommend(ctx, response, userId, category, n, m.RecommendUserBased) case "item_based": - results, err = m.Recommend(response, userId, category, n, m.RecommendItemBased) + results, err = m.Recommend(ctx, response, userId, category, n, m.RecommendItemBased) case "_": recommenders := []server.Recommender{m.RecommendOffline} for _, recommender := range m.Config.Recommend.Online.FallbackRecommend { @@ -687,7 +712,7 @@ func (m *Master) getRecommend(request *restful.Request, response *restful.Respon return } } - results, err = m.Recommend(response, userId, category, n, recommenders...) + results, err = m.Recommend(ctx, response, userId, category, n, recommenders...) } if err != nil { server.InternalServerError(response, err) @@ -696,7 +721,7 @@ func (m *Master) getRecommend(request *restful.Request, response *restful.Respon // Send result details := make([]data.Item, len(results)) for i := range results { - details[i], err = m.DataClient.GetItem(results[i]) + details[i], err = m.DataClient.GetItem(ctx, results[i]) if err != nil { server.InternalServerError(response, err) return @@ -715,9 +740,13 @@ type Feedback struct { // get feedback by user-id with feedback type func (m *Master) getTypedFeedbackByUser(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } feedbackType := request.PathParameter("feedback-type") userId := request.PathParameter("user-id") - feedback, err := m.DataClient.GetUserFeedback(userId, m.Config.Now(), feedbackType) + feedback, err := m.DataClient.GetUserFeedback(ctx, userId, m.Config.Now(), feedbackType) if err != nil { server.InternalServerError(response, err) return @@ -728,7 +757,7 @@ func (m *Master) getTypedFeedbackByUser(request *restful.Request, response *rest details[i].UserId = feedback[i].UserId details[i].Timestamp = feedback[i].Timestamp details[i].Comment = feedback[i].Comment - details[i].Item, err = m.DataClient.GetItem(feedback[i].ItemId) + details[i].Item, err = m.DataClient.GetItem(ctx, feedback[i].ItemId) if errors.Is(err, errors.NotFound) { details[i].Item = data.Item{ItemId: feedback[i].ItemId, Comment: "** This item doesn't exist in Gorse **"} } else if err != nil { @@ -750,6 +779,10 @@ type ScoreUser struct { } func (m *Master) getSort(key, category string, isItem bool, request *restful.Request, response *restful.Response, retType interface{}) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } var n, offset int var err error // read arguments @@ -762,7 +795,7 @@ func (m *Master) getSort(key, category string, isItem bool, request *restful.Req return } // Get the popular list - scores, err := m.CacheClient.GetSorted(cache.Key(key, category), offset, m.Config.Recommend.CacheSize) + scores, err := m.CacheClient.GetSorted(ctx, cache.Key(key, category), offset, m.Config.Recommend.CacheSize) if err != nil { server.InternalServerError(response, err) return @@ -779,7 +812,7 @@ func (m *Master) getSort(key, category string, isItem bool, request *restful.Req details := make([]ScoredItem, len(scores)) for i := range scores { details[i].Score = scores[i].Score - details[i].Item, err = m.DataClient.GetItem(scores[i].Id) + details[i].Item, err = m.DataClient.GetItem(ctx, scores[i].Id) if err != nil { server.InternalServerError(response, err) return @@ -790,7 +823,7 @@ func (m *Master) getSort(key, category string, isItem bool, request *restful.Req details := make([]ScoreUser, len(scores)) for i := range scores { details[i].Score = scores[i].Score - details[i].User, err = m.DataClient.GetUser(scores[i].Id) + details[i].User, err = m.DataClient.GetUser(ctx, scores[i].Id) if err != nil { server.InternalServerError(response, err) return @@ -830,6 +863,10 @@ func (m *Master) getUserNeighbors(request *restful.Request, response *restful.Re } func (m *Master) importExportUsers(response http.ResponseWriter, request *http.Request) { + ctx := context.Background() + if request != nil { + ctx = request.Context() + } if !m.checkLogin(request) { resp := restful.NewResponse(response) err := resp.WriteErrorString(http.StatusUnauthorized, "unauthorized") @@ -850,7 +887,7 @@ func (m *Master) importExportUsers(response http.ResponseWriter, request *http.R return } // write rows - userChan, errChan := m.DataClient.GetUserStream(batchSize) + userChan, errChan := m.DataClient.GetUserStream(ctx, batchSize) for users := range userChan { for _, user := range users { if _, err = response.Write([]byte(fmt.Sprintf("%s,%s\r\n", @@ -880,11 +917,12 @@ func (m *Master) importExportUsers(response http.ResponseWriter, request *http.R return } defer file.Close() - m.importUsers(response, file, hasHeader, sep, labelSep, fmtString) + m.importUsers(ctx, response, file, hasHeader, sep, labelSep, fmtString) } } -func (m *Master) importUsers(response http.ResponseWriter, file io.Reader, hasHeader bool, sep, labelSep, fmtString string) { +func (m *Master) importUsers(ctx context.Context, response http.ResponseWriter, file io.Reader, hasHeader bool, sep, labelSep, fmtString string) { + lineCount := 0 timeStart := time.Now() users := make([]data.User, 0) @@ -921,7 +959,7 @@ func (m *Master) importUsers(response http.ResponseWriter, file io.Reader, hasHe users = append(users, user) // batch insert if len(users) == batchSize { - err = m.DataClient.BatchInsertUsers(users) + err = m.DataClient.BatchInsertUsers(ctx, users) if err != nil { server.InternalServerError(restful.NewResponse(response), err) return false @@ -936,7 +974,7 @@ func (m *Master) importUsers(response http.ResponseWriter, file io.Reader, hasHe return } if len(users) > 0 { - err = m.DataClient.BatchInsertUsers(users) + err = m.DataClient.BatchInsertUsers(ctx, users) if err != nil { server.InternalServerError(restful.NewResponse(response), err) return @@ -951,6 +989,10 @@ func (m *Master) importUsers(response http.ResponseWriter, file io.Reader, hasHe } func (m *Master) importExportItems(response http.ResponseWriter, request *http.Request) { + ctx := context.Background() + if request != nil { + ctx = request.Context() + } if !m.checkLogin(request) { resp := restful.NewResponse(response) err := resp.WriteErrorString(http.StatusUnauthorized, "unauthorized") @@ -971,7 +1013,7 @@ func (m *Master) importExportItems(response http.ResponseWriter, request *http.R return } // write rows - itemChan, errChan := m.DataClient.GetItemStream(batchSize, nil) + itemChan, errChan := m.DataClient.GetItemStream(ctx, batchSize, nil) for items := range itemChan { for _, item := range items { if _, err = response.Write([]byte(fmt.Sprintf("%s,%t,%s,%v,%s,%s\r\n", @@ -1002,13 +1044,13 @@ func (m *Master) importExportItems(response http.ResponseWriter, request *http.R return } defer file.Close() - m.importItems(response, file, hasHeader, sep, labelSep, fmtString) + m.importItems(ctx, response, file, hasHeader, sep, labelSep, fmtString) default: writeError(response, http.StatusMethodNotAllowed, "method not allowed") } } -func (m *Master) importItems(response http.ResponseWriter, file io.Reader, hasHeader bool, sep, labelSep, fmtString string) { +func (m *Master) importItems(ctx context.Context, response http.ResponseWriter, file io.Reader, hasHeader bool, sep, labelSep, fmtString string) { lineCount := 0 timeStart := time.Now() items := make([]data.Item, 0) @@ -1076,7 +1118,7 @@ func (m *Master) importItems(response http.ResponseWriter, file io.Reader, hasHe items = append(items, item) // batch insert if len(items) == batchSize { - err = m.DataClient.BatchInsertItems(items) + err = m.DataClient.BatchInsertItems(ctx, items) if err != nil { server.InternalServerError(restful.NewResponse(response), err) return false @@ -1091,7 +1133,7 @@ func (m *Master) importItems(response http.ResponseWriter, file io.Reader, hasHe return } if len(items) > 0 { - err = m.DataClient.BatchInsertItems(items) + err = m.DataClient.BatchInsertItems(ctx, items) if err != nil { server.InternalServerError(restful.NewResponse(response), err) return @@ -1135,6 +1177,10 @@ func formValue(request *http.Request, fieldName, defaultValue string) string { } func (m *Master) importExportFeedback(response http.ResponseWriter, request *http.Request) { + ctx := context.Background() + if request != nil { + ctx = request.Context() + } if !m.checkLogin(request) { writeError(response, http.StatusUnauthorized, "unauthorized") return @@ -1150,7 +1196,7 @@ func (m *Master) importExportFeedback(response http.ResponseWriter, request *htt return } // write rows - feedbackChan, errChan := m.DataClient.GetFeedbackStream(batchSize, nil, m.Config.Now()) + feedbackChan, errChan := m.DataClient.GetFeedbackStream(ctx, batchSize, nil, m.Config.Now()) for feedback := range feedbackChan { for _, v := range feedback { if _, err = response.Write([]byte(fmt.Sprintf("%s,%s,%s,%v\r\n", @@ -1180,13 +1226,13 @@ func (m *Master) importExportFeedback(response http.ResponseWriter, request *htt return } defer file.Close() - m.importFeedback(response, file, hasHeader, sep, fmtString) + m.importFeedback(ctx, response, file, hasHeader, sep, fmtString) default: writeError(response, http.StatusMethodNotAllowed, "method not allowed") } } -func (m *Master) importFeedback(response http.ResponseWriter, file io.Reader, hasHeader bool, sep, fmtString string) { +func (m *Master) importFeedback(ctx context.Context, response http.ResponseWriter, file io.Reader, hasHeader bool, sep, fmtString string) { var err error scanner := bufio.NewScanner(file) lineCount := 0 @@ -1234,7 +1280,7 @@ func (m *Master) importFeedback(response http.ResponseWriter, file io.Reader, ha feedbacks = append(feedbacks, feedback) // batch insert if len(feedbacks) == batchSize { - err = m.InsertFeedbackToCache(feedbacks) + err = m.InsertFeedbackToCache(ctx, feedbacks) if err != nil { server.InternalServerError(restful.NewResponse(response), err) return false @@ -1249,7 +1295,7 @@ func (m *Master) importFeedback(response http.ResponseWriter, file io.Reader, ha return } // insert to data store - err = m.DataClient.BatchInsertFeedback(feedbacks, + err = m.DataClient.BatchInsertFeedback(ctx, feedbacks, m.Config.Server.AutoInsertUser, m.Config.Server.AutoInsertItem, true) if err != nil { @@ -1258,7 +1304,7 @@ func (m *Master) importFeedback(response http.ResponseWriter, file io.Reader, ha } // insert to cache store if len(feedbacks) > 0 { - err = m.InsertFeedbackToCache(feedbacks) + err = m.InsertFeedbackToCache(ctx, feedbacks) if err != nil { server.InternalServerError(restful.NewResponse(response), err) return diff --git a/master/rest_test.go b/master/rest_test.go index 0919884c3..5a1597ac2 100644 --- a/master/rest_test.go +++ b/master/rest_test.go @@ -16,6 +16,7 @@ package master import ( "bytes" + "context" "encoding/json" "fmt" "github.com/alicebob/miniredis/v2" @@ -105,13 +106,14 @@ func marshal(t *testing.T, v interface{}) string { func TestMaster_ExportUsers(t *testing.T) { s, cookie := newMockServer(t) defer s.Close(t) + ctx := context.Background() // insert users users := []data.User{ {UserId: "1", Labels: []string{"a", "b"}}, {UserId: "2", Labels: []string{"b", "c"}}, {UserId: "3", Labels: []string{"c", "d"}}, } - err := s.DataClient.BatchInsertUsers(users) + err := s.DataClient.BatchInsertUsers(ctx, users) assert.NoError(t, err) // send request req := httptest.NewRequest("GET", "https://example.com/", nil) @@ -130,13 +132,14 @@ func TestMaster_ExportUsers(t *testing.T) { func TestMaster_ExportItems(t *testing.T) { s, cookie := newMockServer(t) defer s.Close(t) + ctx := context.Background() // insert items items := []data.Item{ {"1", false, []string{"x"}, time.Date(2020, 1, 1, 1, 1, 1, 1, time.UTC), []string{"a", "b"}, "o,n,e"}, {"2", false, []string{"x", "y"}, time.Date(2021, 1, 1, 1, 1, 1, 1, time.UTC), []string{"b", "c"}, "t\r\nw\r\no"}, {"3", true, nil, time.Date(2022, 1, 1, 1, 1, 1, 1, time.UTC), nil, "\"three\""}, } - err := s.DataClient.BatchInsertItems(items) + err := s.DataClient.BatchInsertItems(ctx, items) assert.NoError(t, err) // send request req := httptest.NewRequest("GET", "https://example.com/", nil) @@ -155,13 +158,15 @@ func TestMaster_ExportItems(t *testing.T) { func TestMaster_ExportFeedback(t *testing.T) { s, cookie := newMockServer(t) defer s.Close(t) + + ctx := context.Background() // insert feedback feedbacks := []data.Feedback{ {FeedbackKey: data.FeedbackKey{FeedbackType: "click", UserId: "0", ItemId: "2"}}, {FeedbackKey: data.FeedbackKey{FeedbackType: "share", UserId: "1", ItemId: "4"}}, {FeedbackKey: data.FeedbackKey{FeedbackType: "read", UserId: "2", ItemId: "6"}}, } - err := s.DataClient.BatchInsertFeedback(feedbacks, true, true, true) + err := s.DataClient.BatchInsertFeedback(ctx, feedbacks, true, true, true) assert.NoError(t, err) // send request req := httptest.NewRequest("GET", "https://example.com/", nil) @@ -180,6 +185,8 @@ func TestMaster_ExportFeedback(t *testing.T) { func TestMaster_ImportUsers(t *testing.T) { s, cookie := newMockServer(t) defer s.Close(t) + + ctx := context.Background() // send request buf := bytes.NewBuffer(nil) writer := multipart.NewWriter(buf) @@ -207,7 +214,7 @@ func TestMaster_ImportUsers(t *testing.T) { // check assert.Equal(t, http.StatusOK, w.Result().StatusCode) assert.JSONEq(t, marshal(t, server.Success{RowAffected: 3}), w.Body.String()) - _, items, err := s.DataClient.GetUsers("", 100) + _, items, err := s.DataClient.GetUsers(ctx, "", 100) assert.NoError(t, err) assert.Equal(t, []data.User{ {UserId: "1", Labels: []string{"a", "b"}}, @@ -219,6 +226,7 @@ func TestMaster_ImportUsers(t *testing.T) { func TestMaster_ImportUsers_DefaultFormat(t *testing.T) { s, cookie := newMockServer(t) defer s.Close(t) + ctx := context.Background() // send request buf := bytes.NewBuffer(nil) writer := multipart.NewWriter(buf) @@ -239,7 +247,7 @@ func TestMaster_ImportUsers_DefaultFormat(t *testing.T) { // check assert.Equal(t, http.StatusOK, w.Result().StatusCode) assert.JSONEq(t, marshal(t, server.Success{RowAffected: 3}), w.Body.String()) - _, items, err := s.DataClient.GetUsers("", 100) + _, items, err := s.DataClient.GetUsers(ctx, "", 100) assert.NoError(t, err) assert.Equal(t, []data.User{ {UserId: "1", Labels: []string{"a", "用例"}}, @@ -251,6 +259,8 @@ func TestMaster_ImportUsers_DefaultFormat(t *testing.T) { func TestMaster_ImportItems(t *testing.T) { s, cookie := newMockServer(t) defer s.Close(t) + + ctx := context.Background() // send request buf := bytes.NewBuffer(nil) writer := multipart.NewWriter(buf) @@ -278,7 +288,7 @@ func TestMaster_ImportItems(t *testing.T) { // check assert.Equal(t, http.StatusOK, w.Result().StatusCode) assert.JSONEq(t, marshal(t, server.Success{RowAffected: 3}), w.Body.String()) - _, items, err := s.DataClient.GetItems("", 100, nil) + _, items, err := s.DataClient.GetItems(ctx, "", 100, nil) assert.NoError(t, err) assert.Equal(t, []data.Item{ {"1", false, []string{"x"}, time.Date(2020, 1, 1, 1, 1, 1, 1, time.UTC), []string{"a", "b"}, "o,n,e"}, @@ -290,6 +300,8 @@ func TestMaster_ImportItems(t *testing.T) { func TestMaster_ImportItems_DefaultFormat(t *testing.T) { s, cookie := newMockServer(t) defer s.Close(t) + + ctx := context.Background() // send request buf := bytes.NewBuffer(nil) writer := multipart.NewWriter(buf) @@ -310,7 +322,7 @@ func TestMaster_ImportItems_DefaultFormat(t *testing.T) { // check assert.Equal(t, http.StatusOK, w.Result().StatusCode) assert.JSONEq(t, marshal(t, server.Success{RowAffected: 3}), w.Body.String()) - _, items, err := s.DataClient.GetItems("", 100, nil) + _, items, err := s.DataClient.GetItems(ctx, "", 100, nil) assert.NoError(t, err) assert.Equal(t, []data.Item{ {"1", false, []string{"x"}, time.Date(2020, 1, 1, 1, 1, 1, 1, time.UTC), []string{"a", "b"}, "one"}, @@ -322,6 +334,8 @@ func TestMaster_ImportItems_DefaultFormat(t *testing.T) { func TestMaster_ImportFeedback(t *testing.T) { s, cookie := newMockServer(t) defer s.Close(t) + + ctx := context.Background() // send request buf := bytes.NewBuffer(nil) writer := multipart.NewWriter(buf) @@ -347,7 +361,7 @@ func TestMaster_ImportFeedback(t *testing.T) { // check assert.Equal(t, http.StatusOK, w.Result().StatusCode) assert.JSONEq(t, marshal(t, server.Success{RowAffected: 3}), w.Body.String()) - _, feedback, err := s.DataClient.GetFeedback("", 100, nil, lo.ToPtr(time.Now())) + _, feedback, err := s.DataClient.GetFeedback(ctx, "", 100, nil, lo.ToPtr(time.Now())) assert.NoError(t, err) assert.Equal(t, []data.Feedback{ {FeedbackKey: data.FeedbackKey{FeedbackType: "click", UserId: "0", ItemId: "2"}}, @@ -360,6 +374,7 @@ func TestMaster_ImportFeedback_Default(t *testing.T) { s, cookie := newMockServer(t) defer s.Close(t) // send request + ctx := context.Background() buf := bytes.NewBuffer(nil) writer := multipart.NewWriter(buf) file, err := writer.CreateFormFile("file", "feedback.csv") @@ -379,7 +394,7 @@ func TestMaster_ImportFeedback_Default(t *testing.T) { // check assert.Equal(t, http.StatusOK, w.Result().StatusCode) assert.JSONEq(t, marshal(t, server.Success{RowAffected: 3}), w.Body.String()) - _, feedback, err := s.DataClient.GetFeedback("", 100, nil, lo.ToPtr(time.Now())) + _, feedback, err := s.DataClient.GetFeedback(ctx, "", 100, nil, lo.ToPtr(time.Now())) assert.NoError(t, err) assert.Equal(t, []data.Feedback{ {FeedbackKey: data.FeedbackKey{FeedbackType: "click", UserId: "0", ItemId: "2"}}, @@ -411,16 +426,18 @@ func TestMaster_GetCluster(t *testing.T) { func TestMaster_GetStats(t *testing.T) { s, cookie := newMockServer(t) defer s.Close(t) + + ctx := context.Background() // set stats s.rankingScore = ranking.Score{Precision: 0.1} s.clickScore = click.Score{Precision: 0.2} - err := s.CacheClient.Set(cache.Integer(cache.Key(cache.GlobalMeta, cache.NumUsers), 123)) + err := s.CacheClient.Set(ctx, cache.Integer(cache.Key(cache.GlobalMeta, cache.NumUsers), 123)) assert.NoError(t, err) - err = s.CacheClient.Set(cache.Integer(cache.Key(cache.GlobalMeta, cache.NumItems), 234)) + err = s.CacheClient.Set(ctx, cache.Integer(cache.Key(cache.GlobalMeta, cache.NumItems), 234)) assert.NoError(t, err) - err = s.CacheClient.Set(cache.Integer(cache.Key(cache.GlobalMeta, cache.NumValidPosFeedbacks), 345)) + err = s.CacheClient.Set(ctx, cache.Integer(cache.Key(cache.GlobalMeta, cache.NumValidPosFeedbacks), 345)) assert.NoError(t, err) - err = s.CacheClient.Set(cache.Integer(cache.Key(cache.GlobalMeta, cache.NumValidNegFeedbacks), 456)) + err = s.CacheClient.Set(ctx, cache.Integer(cache.Key(cache.GlobalMeta, cache.NumValidNegFeedbacks), 456)) assert.NoError(t, err) // get stats apitest.New(). @@ -445,22 +462,23 @@ func TestMaster_GetRates(t *testing.T) { s, cookie := newMockServer(t) defer s.Close(t) + ctx := context.Background() // write rates s.Config.Recommend.DataSource.PositiveFeedbackTypes = []string{"a", "b"} // This first measurement should be overwritten. - err := s.RestServer.InsertMeasurement(server.Measurement{Name: cache.Key(PositiveFeedbackRate, "a"), Value: 100.0, Timestamp: time.Date(2000, 1, 1, 1, 1, 1, 0, time.UTC)}) + err := s.RestServer.InsertMeasurement(ctx, server.Measurement{Name: cache.Key(PositiveFeedbackRate, "a"), Value: 100.0, Timestamp: time.Date(2000, 1, 1, 1, 1, 1, 0, time.UTC)}) assert.NoError(t, err) - err = s.RestServer.InsertMeasurement(server.Measurement{Name: cache.Key(PositiveFeedbackRate, "a"), Value: 2.0, Timestamp: time.Date(2000, 1, 1, 1, 1, 1, 0, time.UTC)}) + err = s.RestServer.InsertMeasurement(ctx, server.Measurement{Name: cache.Key(PositiveFeedbackRate, "a"), Value: 2.0, Timestamp: time.Date(2000, 1, 1, 1, 1, 1, 0, time.UTC)}) assert.NoError(t, err) - err = s.RestServer.InsertMeasurement(server.Measurement{Name: cache.Key(PositiveFeedbackRate, "a"), Value: 2.0, Timestamp: time.Date(2000, 1, 2, 1, 1, 1, 0, time.UTC)}) + err = s.RestServer.InsertMeasurement(ctx, server.Measurement{Name: cache.Key(PositiveFeedbackRate, "a"), Value: 2.0, Timestamp: time.Date(2000, 1, 2, 1, 1, 1, 0, time.UTC)}) assert.NoError(t, err) - err = s.RestServer.InsertMeasurement(server.Measurement{Name: cache.Key(PositiveFeedbackRate, "a"), Value: 3.0, Timestamp: time.Date(2000, 1, 3, 1, 1, 1, 0, time.UTC)}) + err = s.RestServer.InsertMeasurement(ctx, server.Measurement{Name: cache.Key(PositiveFeedbackRate, "a"), Value: 3.0, Timestamp: time.Date(2000, 1, 3, 1, 1, 1, 0, time.UTC)}) assert.NoError(t, err) - err = s.RestServer.InsertMeasurement(server.Measurement{Name: cache.Key(PositiveFeedbackRate, "b"), Value: 20.0, Timestamp: time.Date(2000, 1, 1, 1, 1, 1, 0, time.UTC)}) + err = s.RestServer.InsertMeasurement(ctx, server.Measurement{Name: cache.Key(PositiveFeedbackRate, "b"), Value: 20.0, Timestamp: time.Date(2000, 1, 1, 1, 1, 1, 0, time.UTC)}) assert.NoError(t, err) - err = s.RestServer.InsertMeasurement(server.Measurement{Name: cache.Key(PositiveFeedbackRate, "b"), Value: 20.0, Timestamp: time.Date(2000, 1, 2, 1, 1, 1, 0, time.UTC)}) + err = s.RestServer.InsertMeasurement(ctx, server.Measurement{Name: cache.Key(PositiveFeedbackRate, "b"), Value: 20.0, Timestamp: time.Date(2000, 1, 2, 1, 1, 1, 0, time.UTC)}) assert.NoError(t, err) - err = s.RestServer.InsertMeasurement(server.Measurement{Name: cache.Key(PositiveFeedbackRate, "b"), Value: 30.0, Timestamp: time.Date(2000, 1, 3, 1, 1, 1, 0, time.UTC)}) + err = s.RestServer.InsertMeasurement(ctx, server.Measurement{Name: cache.Key(PositiveFeedbackRate, "b"), Value: 30.0, Timestamp: time.Date(2000, 1, 3, 1, 1, 1, 0, time.UTC)}) assert.NoError(t, err) // get rates @@ -488,8 +506,9 @@ func TestMaster_GetRates(t *testing.T) { func TestMaster_GetCategories(t *testing.T) { s, cookie := newMockServer(t) defer s.Close(t) + ctx := context.Background() // insert categories - err := s.CacheClient.SetSet(cache.ItemCategories, "a", "b", "c") + err := s.CacheClient.SetSet(ctx, cache.ItemCategories, "a", "b", "c") assert.NoError(t, err) // get categories apitest.New(). @@ -505,6 +524,7 @@ func TestMaster_GetCategories(t *testing.T) { func TestMaster_GetUsers(t *testing.T) { s, cookie := newMockServer(t) defer s.Close(t) + ctx := context.Background() // add users users := []User{ {data.User{UserId: "0"}, time.Date(2000, 1, 1, 1, 1, 1, 1, time.UTC), time.Date(2020, 1, 1, 1, 1, 1, 1, time.UTC)}, @@ -512,11 +532,11 @@ func TestMaster_GetUsers(t *testing.T) { {data.User{UserId: "2"}, time.Date(2002, 1, 1, 1, 1, 1, 1, time.UTC), time.Date(2022, 1, 1, 1, 1, 1, 1, time.UTC)}, } for _, user := range users { - err := s.DataClient.BatchInsertUsers([]data.User{user.User}) + err := s.DataClient.BatchInsertUsers(ctx, []data.User{user.User}) assert.NoError(t, err) - err = s.CacheClient.Set(cache.Time(cache.Key(cache.LastModifyUserTime, user.UserId), user.LastActiveTime)) + err = s.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastModifyUserTime, user.UserId), user.LastActiveTime)) assert.NoError(t, err) - err = s.CacheClient.Set(cache.Time(cache.Key(cache.LastUpdateUserRecommendTime, user.UserId), user.LastUpdateTime)) + err = s.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastUpdateUserRecommendTime, user.UserId), user.LastUpdateTime)) assert.NoError(t, err) } // get users @@ -551,6 +571,7 @@ func TestServer_SortedItems(t *testing.T) { Label string Get string } + ctx := context.Background() operators := []ListOperator{ {"Item Neighbors", cache.ItemNeighbors, "0", "/api/dashboard/item/0/neighbors"}, {"Item Neighbors in Category", cache.ItemNeighbors, "0/*", "/api/dashboard/item/0/neighbors/*"}, @@ -569,14 +590,14 @@ func TestServer_SortedItems(t *testing.T) { {strconv.Itoa(i) + "3", 97}, {strconv.Itoa(i) + "4", 96}, } - err := s.CacheClient.SetSorted(cache.Key(operator.Prefix, operator.Label), scores) + err := s.CacheClient.SetSorted(ctx, cache.Key(operator.Prefix, operator.Label), scores) assert.NoError(t, err) err = server.NewCacheModification(s.CacheClient, s.HiddenItemsManager).HideItem(strconv.Itoa(i) + "3").Exec() assert.NoError(t, err) items := make([]ScoredItem, 0) for _, score := range scores { items = append(items, ScoredItem{Item: data.Item{ItemId: score.Id}, Score: score.Score}) - err = s.DataClient.BatchInsertItems([]data.Item{{ItemId: score.Id}}) + err = s.DataClient.BatchInsertItems(ctx, []data.Item{{ItemId: score.Id}}) assert.NoError(t, err) } apitest.New(). @@ -599,6 +620,7 @@ func TestServer_SortedUsers(t *testing.T) { Label string Get string } + ctx := context.Background() operators := []ListOperator{ {cache.UserNeighbors, "0", "/api/dashboard/user/0/neighbors"}, } @@ -612,12 +634,12 @@ func TestServer_SortedUsers(t *testing.T) { {"3", 97}, {"4", 96}, } - err := s.CacheClient.SetSorted(cache.Key(operator.Prefix, operator.Label), scores) + err := s.CacheClient.SetSorted(ctx, cache.Key(operator.Prefix, operator.Label), scores) assert.NoError(t, err) users := make([]ScoreUser, 0) for _, score := range scores { users = append(users, ScoreUser{User: data.User{UserId: score.Id}, Score: score.Score}) - err = s.DataClient.BatchInsertUsers([]data.User{{UserId: score.Id}}) + err = s.DataClient.BatchInsertUsers(ctx, []data.User{{UserId: score.Id}}) assert.NoError(t, err) } apitest.New(). @@ -634,6 +656,7 @@ func TestServer_SortedUsers(t *testing.T) { func TestServer_Feedback(t *testing.T) { s, cookie := newMockServer(t) defer s.Close(t) + ctx := context.Background() // insert feedback feedback := []Feedback{ {FeedbackType: "click", UserId: "0", Item: data.Item{ItemId: "0"}}, @@ -643,7 +666,7 @@ func TestServer_Feedback(t *testing.T) { {FeedbackType: "click", UserId: "0", Item: data.Item{ItemId: "8"}}, } for _, v := range feedback { - err := s.DataClient.BatchInsertFeedback([]data.Feedback{{ + err := s.DataClient.BatchInsertFeedback(ctx, []data.Feedback{{ FeedbackKey: data.FeedbackKey{FeedbackType: v.FeedbackType, UserId: v.UserId, ItemId: v.Item.ItemId}, }}, true, true, true) assert.NoError(t, err) @@ -673,18 +696,19 @@ func TestServer_GetRecommends(t *testing.T) { {"7", 93}, {"8", 92}, } - err := s.CacheClient.SetSorted(cache.Key(cache.OfflineRecommend, "0"), itemIds) + ctx := context.Background() + err := s.CacheClient.SetSorted(ctx, cache.Key(cache.OfflineRecommend, "0"), itemIds) assert.NoError(t, err) // insert feedback feedback := []data.Feedback{ {FeedbackKey: data.FeedbackKey{FeedbackType: "a", UserId: "0", ItemId: "2"}}, {FeedbackKey: data.FeedbackKey{FeedbackType: "a", UserId: "0", ItemId: "4"}}, } - err = s.RestServer.InsertFeedbackToCache(feedback) + err = s.RestServer.InsertFeedbackToCache(ctx, feedback) assert.NoError(t, err) // insert items for _, item := range itemIds { - err = s.DataClient.BatchInsertItems([]data.Item{{ItemId: item.Id}}) + err = s.DataClient.BatchInsertItems(ctx, []data.Item{{ItemId: item.Id}}) assert.NoError(t, err) } apitest.New(). @@ -715,26 +739,27 @@ func TestMaster_Purge(t *testing.T) { s, cookie := newMockServer(t) defer s.Close(t) + ctx := context.Background() // insert data - err := s.CacheClient.Set(cache.String("key", "value")) + err := s.CacheClient.Set(ctx, cache.String("key", "value")) assert.NoError(t, err) - ret, err := s.CacheClient.Get("key").String() + ret, err := s.CacheClient.Get(ctx, "key").String() assert.NoError(t, err) assert.Equal(t, "value", ret) - err = s.CacheClient.AddSet("set", "a", "b", "c") + err = s.CacheClient.AddSet(ctx, "set", "a", "b", "c") assert.NoError(t, err) - set, err := s.CacheClient.GetSet("set") + set, err := s.CacheClient.GetSet(ctx, "set") assert.NoError(t, err) assert.ElementsMatch(t, []string{"a", "b", "c"}, set) - err = s.CacheClient.AddSorted(cache.Sorted("sorted", []cache.Scored{{Id: "a", Score: 1}, {Id: "b", Score: 2}, {Id: "c", Score: 3}})) + err = s.CacheClient.AddSorted(ctx, cache.Sorted("sorted", []cache.Scored{{Id: "a", Score: 1}, {Id: "b", Score: 2}, {Id: "c", Score: 3}})) assert.NoError(t, err) - z, err := s.CacheClient.GetSorted("sorted", 0, -1) + z, err := s.CacheClient.GetSorted(ctx, "sorted", 0, -1) assert.NoError(t, err) assert.ElementsMatch(t, []cache.Scored{{Id: "a", Score: 1}, {Id: "b", Score: 2}, {Id: "c", Score: 3}}, z) - err = s.DataClient.BatchInsertFeedback(lo.Map(lo.Range(100), func(t int, i int) data.Feedback { + err = s.DataClient.BatchInsertFeedback(ctx, lo.Map(lo.Range(100), func(t int, i int) data.Feedback { return data.Feedback{FeedbackKey: data.FeedbackKey{ FeedbackType: "click", UserId: strconv.Itoa(t), @@ -742,13 +767,13 @@ func TestMaster_Purge(t *testing.T) { }} }), true, true, true) assert.NoError(t, err) - _, users, err := s.DataClient.GetUsers("", 100) + _, users, err := s.DataClient.GetUsers(ctx, "", 100) assert.NoError(t, err) assert.Equal(t, 100, len(users)) - _, items, err := s.DataClient.GetItems("", 100, nil) + _, items, err := s.DataClient.GetItems(ctx, "", 100, nil) assert.NoError(t, err) assert.Equal(t, 100, len(items)) - _, feedbacks, err := s.DataClient.GetFeedback("", 100, nil, lo.ToPtr(time.Now())) + _, feedbacks, err := s.DataClient.GetFeedback(ctx, "", 100, nil, lo.ToPtr(time.Now())) assert.NoError(t, err) assert.Equal(t, 100, len(feedbacks)) @@ -761,22 +786,22 @@ func TestMaster_Purge(t *testing.T) { s.purge(w, req) assert.Equal(t, http.StatusOK, w.Code) - _, err = s.CacheClient.Get("key").String() + _, err = s.CacheClient.Get(ctx, "key").String() assert.ErrorIs(t, err, errors.NotFound) - set, err = s.CacheClient.GetSet("set") + set, err = s.CacheClient.GetSet(ctx, "set") assert.NoError(t, err) assert.Empty(t, set) - z, err = s.CacheClient.GetSorted("sorted", 0, -1) + z, err = s.CacheClient.GetSorted(ctx, "sorted", 0, -1) assert.NoError(t, err) assert.Empty(t, z) - _, users, err = s.DataClient.GetUsers("", 100) + _, users, err = s.DataClient.GetUsers(ctx, "", 100) assert.NoError(t, err) assert.Empty(t, users) - _, items, err = s.DataClient.GetItems("", 100, nil) + _, items, err = s.DataClient.GetItems(ctx, "", 100, nil) assert.NoError(t, err) assert.Empty(t, items) - _, feedbacks, err = s.DataClient.GetFeedback("", 100, nil, lo.ToPtr(time.Now())) + _, feedbacks, err = s.DataClient.GetFeedback(ctx, "", 100, nil, lo.ToPtr(time.Now())) assert.NoError(t, err) assert.Empty(t, feedbacks) } diff --git a/master/tasks.go b/master/tasks.go index abfbaaeaa..5c3c8ba58 100644 --- a/master/tasks.go +++ b/master/tasks.go @@ -15,6 +15,7 @@ package master import ( + "context" "fmt" "math" "sort" @@ -66,6 +67,7 @@ type Task interface { // runLoadDatasetTask loads dataset. func (m *Master) runLoadDatasetTask() error { + ctx := context.Background() initialStartTime := time.Now() log.Logger().Info("load dataset", zap.Strings("positive_feedback_types", m.Config.Recommend.DataSource.PositiveFeedbackTypes), @@ -85,65 +87,65 @@ func (m *Master) runLoadDatasetTask() error { // save popular items to cache for category, items := range popularItems { - if err = m.CacheClient.SetSorted(cache.Key(cache.PopularItems, category), items); err != nil { + if err = m.CacheClient.SetSorted(ctx, cache.Key(cache.PopularItems, category), items); err != nil { log.Logger().Error("failed to cache popular items", zap.Error(err)) } } - if err = m.CacheClient.Set(cache.Time(cache.Key(cache.GlobalMeta, cache.LastUpdatePopularItemsTime), time.Now())); err != nil { + if err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.GlobalMeta, cache.LastUpdatePopularItemsTime), time.Now())); err != nil { log.Logger().Error("failed to write latest update popular items time", zap.Error(err)) } // save the latest items to cache for category, items := range latestItems { - if err = m.CacheClient.AddSorted(cache.Sorted(cache.Key(cache.LatestItems, category), items)); err != nil { + if err = m.CacheClient.AddSorted(ctx, cache.Sorted(cache.Key(cache.LatestItems, category), items)); err != nil { log.Logger().Error("failed to cache latest items", zap.Error(err)) } // reclaim outdated items if len(items) > 0 { threshold := items[len(items)-1].Score - 1 - if err = m.CacheClient.RemSortedByScore(cache.Key(cache.LatestItems, category), math.Inf(-1), threshold); err != nil { + if err = m.CacheClient.RemSortedByScore(ctx, cache.Key(cache.LatestItems, category), math.Inf(-1), threshold); err != nil { log.Logger().Error("failed to reclaim outdated items", zap.Error(err)) } } } - if err = m.CacheClient.Set(cache.Time(cache.Key(cache.GlobalMeta, cache.LastUpdateLatestItemsTime), time.Now())); err != nil { + if err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.GlobalMeta, cache.LastUpdateLatestItemsTime), time.Now())); err != nil { log.Logger().Error("failed to write latest update latest items time", zap.Error(err)) } // write statistics to database UsersTotal.Set(float64(rankingDataset.UserCount())) - if err = m.CacheClient.Set(cache.Integer(cache.Key(cache.GlobalMeta, cache.NumUsers), rankingDataset.UserCount())); err != nil { + if err = m.CacheClient.Set(ctx, cache.Integer(cache.Key(cache.GlobalMeta, cache.NumUsers), rankingDataset.UserCount())); err != nil { log.Logger().Error("failed to write number of users", zap.Error(err)) } ItemsTotal.Set(float64(rankingDataset.ItemCount())) - if err = m.CacheClient.Set(cache.Integer(cache.Key(cache.GlobalMeta, cache.NumItems), rankingDataset.ItemCount())); err != nil { + if err = m.CacheClient.Set(ctx, cache.Integer(cache.Key(cache.GlobalMeta, cache.NumItems), rankingDataset.ItemCount())); err != nil { log.Logger().Error("failed to write number of items", zap.Error(err)) } ImplicitFeedbacksTotal.Set(float64(rankingDataset.Count())) - if err = m.CacheClient.Set(cache.Integer(cache.Key(cache.GlobalMeta, cache.NumTotalPosFeedbacks), rankingDataset.Count())); err != nil { + if err = m.CacheClient.Set(ctx, cache.Integer(cache.Key(cache.GlobalMeta, cache.NumTotalPosFeedbacks), rankingDataset.Count())); err != nil { log.Logger().Error("failed to write number of positive feedbacks", zap.Error(err)) } UserLabelsTotal.Set(float64(clickDataset.Index.CountUserLabels())) - if err = m.CacheClient.Set(cache.Integer(cache.Key(cache.GlobalMeta, cache.NumUserLabels), int(clickDataset.Index.CountUserLabels()))); err != nil { + if err = m.CacheClient.Set(ctx, cache.Integer(cache.Key(cache.GlobalMeta, cache.NumUserLabels), int(clickDataset.Index.CountUserLabels()))); err != nil { log.Logger().Error("failed to write number of user labels", zap.Error(err)) } ItemLabelsTotal.Set(float64(clickDataset.Index.CountItemLabels())) - if err = m.CacheClient.Set(cache.Integer(cache.Key(cache.GlobalMeta, cache.NumItemLabels), int(clickDataset.Index.CountItemLabels()))); err != nil { + if err = m.CacheClient.Set(ctx, cache.Integer(cache.Key(cache.GlobalMeta, cache.NumItemLabels), int(clickDataset.Index.CountItemLabels()))); err != nil { log.Logger().Error("failed to write number of item labels", zap.Error(err)) } ImplicitFeedbacksTotal.Set(float64(rankingDataset.Count())) PositiveFeedbacksTotal.Set(float64(clickDataset.PositiveCount)) - if err = m.CacheClient.Set(cache.Integer(cache.Key(cache.GlobalMeta, cache.NumValidPosFeedbacks), clickDataset.PositiveCount)); err != nil { + if err = m.CacheClient.Set(ctx, cache.Integer(cache.Key(cache.GlobalMeta, cache.NumValidPosFeedbacks), clickDataset.PositiveCount)); err != nil { log.Logger().Error("failed to write number of positive feedbacks", zap.Error(err)) } NegativeFeedbackTotal.Set(float64(clickDataset.NegativeCount)) - if err = m.CacheClient.Set(cache.Integer(cache.Key(cache.GlobalMeta, cache.NumValidNegFeedbacks), clickDataset.NegativeCount)); err != nil { + if err = m.CacheClient.Set(ctx, cache.Integer(cache.Key(cache.GlobalMeta, cache.NumValidNegFeedbacks), clickDataset.NegativeCount)); err != nil { log.Logger().Error("failed to write number of negative feedbacks", zap.Error(err)) } // evaluate positive feedback rate measurement := evaluator.Evaluate() - if err = m.RestServer.InsertMeasurement(measurement...); err != nil { + if err = m.RestServer.InsertMeasurement(ctx, measurement...); err != nil { log.Logger().Error("failed to insert measurement", zap.Error(err)) } @@ -169,7 +171,7 @@ func (m *Master) runLoadDatasetTask() error { InactiveItemsTotal.Set(float64(inactiveItems)) // write categories to cache - if err = m.CacheClient.SetSet(cache.ItemCategories, rankingDataset.CategorySet.List()...); err != nil { + if err = m.CacheClient.SetSet(ctx, cache.ItemCategories, rankingDataset.CategorySet.List()...); err != nil { log.Logger().Error("failed to write categories to cache", zap.Error(err)) } @@ -238,6 +240,7 @@ func (t *FindItemNeighborsTask) run(j *task.JobsAllocator) error { dataset := t.rankingTrainSet numItems := dataset.ItemCount() numFeedback := dataset.Count() + ctx := context.Background() if numItems == 0 { t.taskMonitor.Fail(TaskFindItemNeighbors, "No item found.") @@ -331,7 +334,7 @@ func (t *FindItemNeighborsTask) run(j *task.JobsAllocator) error { t.taskMonitor.Fail(TaskFindItemNeighbors, err.Error()) FindItemNeighborsTotalSeconds.Set(0) } else { - if err := t.CacheClient.Set(cache.Time(cache.Key(cache.GlobalMeta, cache.LastUpdateItemNeighborsTime), time.Now())); err != nil { + if err := t.CacheClient.Set(ctx, cache.Time(cache.Key(cache.GlobalMeta, cache.LastUpdateItemNeighborsTime), time.Now())); err != nil { log.Logger().Error("failed to set neighbors of items update time", zap.Error(err)) } log.Logger().Info("complete searching neighbors of items", @@ -347,6 +350,7 @@ func (t *FindItemNeighborsTask) run(j *task.JobsAllocator) error { func (m *Master) findItemNeighborsBruteForce(dataset *ranking.DataSet, labeledItems [][]int32, labelIDF, userIDF []float32, completed chan struct{}, j *task.JobsAllocator) error { + ctx := context.Background() var ( updateItemCount atomic.Float64 findNeighborSeconds atomic.Float64 @@ -401,12 +405,13 @@ func (m *Master) findItemNeighborsBruteForce(dataset *ranking.DataSet, labeledIt for i := range recommends { recommends[i] = dataset.ItemIndex.ToName(elem[i]) } - if err := m.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, itemId, category), + if err := m.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, itemId, category), cache.CreateScoredItems(recommends, scores)); err != nil { return errors.Trace(err) } } if err := m.CacheClient.Set( + ctx, cache.Time(cache.Key(cache.LastUpdateItemNeighborsTime, itemId), time.Now()), cache.String(cache.Key(cache.ItemNeighborsDigest, itemId), m.Config.ItemNeighborDigest())); err != nil { return errors.Trace(err) @@ -430,6 +435,7 @@ func (m *Master) findItemNeighborsIVF(dataset *ranking.DataSet, labelIDF, userID findNeighborSeconds atomic.Float64 buildIndexSeconds atomic.Float64 ) + ctx := context.Background() // build index buildStart := time.Now() @@ -460,7 +466,7 @@ func (m *Master) findItemNeighborsIVF(dataset *ranking.DataSet, labelIDF, userID true, m.taskMonitor.GetTask(TaskFindItemNeighbors)) ItemNeighborIndexRecall.Set(float64(recall)) - if err := m.CacheClient.Set(cache.String(cache.Key(cache.GlobalMeta, cache.ItemNeighborIndexRecall), encoding.FormatFloat32(recall))); err != nil { + if err := m.CacheClient.Set(ctx, cache.String(cache.Key(cache.GlobalMeta, cache.ItemNeighborIndexRecall), encoding.FormatFloat32(recall))); err != nil { return errors.Trace(err) } buildIndexSeconds.Add(time.Since(buildStart).Seconds()) @@ -494,12 +500,13 @@ func (m *Master) findItemNeighborsIVF(dataset *ranking.DataSet, labelIDF, userID itemScores[i].Id = dataset.ItemIndex.ToName(neighbors[category][i]) itemScores[i].Score = float64(-scores[category][i]) } - if err := m.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, itemId, category), itemScores); err != nil { + if err := m.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, itemId, category), itemScores); err != nil { return errors.Trace(err) } } } if err := m.CacheClient.Set( + ctx, cache.Time(cache.Key(cache.LastUpdateItemNeighborsTime, itemId), time.Now()), cache.String(cache.Key(cache.ItemNeighborsDigest, itemId), m.Config.ItemNeighborDigest())); err != nil { return errors.Trace(err) @@ -557,6 +564,7 @@ func (t *FindUserNeighborsTask) run(j *task.JobsAllocator) error { dataset := t.rankingTrainSet numUsers := dataset.UserCount() numFeedback := dataset.Count() + ctx := context.Background() if numUsers == 0 { t.taskMonitor.Fail(TaskFindItemNeighbors, "No item found.") @@ -650,7 +658,7 @@ func (t *FindUserNeighborsTask) run(j *task.JobsAllocator) error { t.taskMonitor.Fail(TaskFindUserNeighbors, err.Error()) FindUserNeighborsTotalSeconds.Set(0) } else { - if err := t.CacheClient.Set(cache.Time(cache.Key(cache.GlobalMeta, cache.LastUpdateUserNeighborsTime), time.Now())); err != nil { + if err := t.CacheClient.Set(ctx, cache.Time(cache.Key(cache.GlobalMeta, cache.LastUpdateUserNeighborsTime), time.Now())); err != nil { log.Logger().Error("failed to set neighbors of users update time", zap.Error(err)) } log.Logger().Info("complete searching neighbors of users", @@ -669,6 +677,7 @@ func (m *Master) findUserNeighborsBruteForce(dataset *ranking.DataSet, labeledUs updateUserCount atomic.Float64 findNeighborSeconds atomic.Float64 ) + ctx := context.Background() var vectors VectorsInterface switch m.Config.Recommend.UserNeighbors.NeighborType { @@ -711,11 +720,12 @@ func (m *Master) findUserNeighborsBruteForce(dataset *ranking.DataSet, labeledUs for i := range recommends { recommends[i] = dataset.UserIndex.ToName(elem[i]) } - if err := m.CacheClient.SetSorted(cache.Key(cache.UserNeighbors, userId), + if err := m.CacheClient.SetSorted(ctx, cache.Key(cache.UserNeighbors, userId), cache.CreateScoredItems(recommends, scores)); err != nil { return errors.Trace(err) } if err := m.CacheClient.Set( + ctx, cache.Time(cache.Key(cache.LastUpdateUserNeighborsTime, userId), time.Now()), cache.String(cache.Key(cache.UserNeighborsDigest, userId), m.Config.UserNeighborDigest())); err != nil { return errors.Trace(err) @@ -739,7 +749,7 @@ func (m *Master) findUserNeighborsIVF(dataset *ranking.DataSet, labelIDF, itemID buildIndexSeconds atomic.Float64 findNeighborSeconds atomic.Float64 ) - + ctx := context.Background() // build index buildStart := time.Now() var index search.VectorIndex @@ -771,7 +781,7 @@ func (m *Master) findUserNeighborsIVF(dataset *ranking.DataSet, labelIDF, itemID true, m.taskMonitor.GetTask(TaskFindUserNeighbors)) UserNeighborIndexRecall.Set(float64(recall)) - if err := m.CacheClient.Set(cache.String(cache.Key(cache.GlobalMeta, cache.UserNeighborIndexRecall), encoding.FormatFloat32(recall))); err != nil { + if err := m.CacheClient.Set(ctx, cache.String(cache.Key(cache.GlobalMeta, cache.UserNeighborIndexRecall), encoding.FormatFloat32(recall))); err != nil { return errors.Trace(err) } buildIndexSeconds.Add(time.Since(buildStart).Seconds()) @@ -794,10 +804,11 @@ func (m *Master) findUserNeighborsIVF(dataset *ranking.DataSet, labelIDF, itemID itemScores[i].Id = dataset.UserIndex.ToName(neighbors[i]) itemScores[i].Score = float64(-scores[i]) } - if err := m.CacheClient.SetSorted(cache.Key(cache.UserNeighbors, userId), itemScores); err != nil { + if err := m.CacheClient.SetSorted(ctx, cache.Key(cache.UserNeighbors, userId), itemScores); err != nil { return errors.Trace(err) } if err := m.CacheClient.Set( + ctx, cache.Time(cache.Key(cache.LastUpdateUserNeighborsTime, userId), time.Now()), cache.String(cache.Key(cache.UserNeighborsDigest, userId), m.Config.UserNeighborDigest())); err != nil { return errors.Trace(err) @@ -849,15 +860,16 @@ func (m *Master) checkUserNeighborCacheTimeout(userId string) bool { cacheDigest string err error ) + ctx := context.Background() // check cache - if items, err := m.CacheClient.GetSorted(cache.Key(cache.UserNeighbors, userId), 0, -1); err != nil { + if items, err := m.CacheClient.GetSorted(ctx, cache.Key(cache.UserNeighbors, userId), 0, -1); err != nil { log.Logger().Error("failed to load user neighbors", zap.String("user_id", userId), zap.Error(err)) return true } else if len(items) == 0 { return true } // read digest - cacheDigest, err = m.CacheClient.Get(cache.Key(cache.UserNeighborsDigest, userId)).String() + cacheDigest, err = m.CacheClient.Get(ctx, cache.Key(cache.UserNeighborsDigest, userId)).String() if err != nil { if !errors.Is(err, errors.NotFound) { log.Logger().Error("failed to read user neighbors digest", zap.Error(err)) @@ -868,7 +880,7 @@ func (m *Master) checkUserNeighborCacheTimeout(userId string) bool { return true } // read modified time - modifiedTime, err = m.CacheClient.Get(cache.Key(cache.LastModifyUserTime, userId)).Time() + modifiedTime, err = m.CacheClient.Get(ctx, cache.Key(cache.LastModifyUserTime, userId)).Time() if err != nil { if !errors.Is(err, errors.NotFound) { log.Logger().Error("failed to read last modify user time", zap.Error(err)) @@ -876,7 +888,7 @@ func (m *Master) checkUserNeighborCacheTimeout(userId string) bool { return true } // read update time - updateTime, err = m.CacheClient.Get(cache.Key(cache.LastUpdateUserNeighborsTime, userId)).Time() + updateTime, err = m.CacheClient.Get(ctx, cache.Key(cache.LastUpdateUserNeighborsTime, userId)).Time() if err != nil { if !errors.Is(err, errors.NotFound) { log.Logger().Error("failed to read last update user neighbors time", zap.Error(err)) @@ -901,9 +913,11 @@ func (m *Master) checkItemNeighborCacheTimeout(itemId string, categories []strin cacheDigest string err error ) + ctx := context.Background() + // check cache for _, category := range append([]string{""}, categories...) { - items, err := m.CacheClient.GetSorted(cache.Key(cache.ItemNeighbors, itemId, category), 0, -1) + items, err := m.CacheClient.GetSorted(ctx, cache.Key(cache.ItemNeighbors, itemId, category), 0, -1) if err != nil { log.Logger().Error("failed to load item neighbors", zap.String("item_id", itemId), zap.Error(err)) return true @@ -912,7 +926,7 @@ func (m *Master) checkItemNeighborCacheTimeout(itemId string, categories []strin } } // read digest - cacheDigest, err = m.CacheClient.Get(cache.Key(cache.ItemNeighborsDigest, itemId)).String() + cacheDigest, err = m.CacheClient.Get(ctx, cache.Key(cache.ItemNeighborsDigest, itemId)).String() if err != nil { if !errors.Is(err, errors.NotFound) { log.Logger().Error("failed to read item neighbors digest", zap.Error(err)) @@ -923,7 +937,7 @@ func (m *Master) checkItemNeighborCacheTimeout(itemId string, categories []strin return true } // read modified time - modifiedTime, err = m.CacheClient.Get(cache.Key(cache.LastModifyItemTime, itemId)).Time() + modifiedTime, err = m.CacheClient.Get(ctx, cache.Key(cache.LastModifyItemTime, itemId)).Time() if err != nil { if !errors.Is(err, errors.NotFound) { log.Logger().Error("failed to read last modify item time", zap.Error(err)) @@ -931,7 +945,7 @@ func (m *Master) checkItemNeighborCacheTimeout(itemId string, categories []strin return true } // read update time - updateTime, err = m.CacheClient.Get(cache.Key(cache.LastUpdateItemNeighborsTime, itemId)).Time() + updateTime, err = m.CacheClient.Get(ctx, cache.Key(cache.LastUpdateItemNeighborsTime, itemId)).Time() if err != nil { if !errors.Is(err, errors.NotFound) { log.Logger().Error("failed to read last update item neighbors time", zap.Error(err)) @@ -970,6 +984,7 @@ func (t *FitRankingModelTask) run(j *task.JobsAllocator) error { numFeedback := dataset.Count() var modelChanged bool + ctx := context.Background() bestRankingName, bestRankingModel, bestRankingScore := t.rankingModelSearcher.GetBestModel() t.rankingModelMutex.Lock() if bestRankingModel != nil && !bestRankingModel.Invalid() && @@ -1016,7 +1031,7 @@ func (t *FitRankingModelTask) run(j *task.JobsAllocator) error { CollaborativeFilteringRecall10.Set(float64(score.Recall)) CollaborativeFilteringPrecision10.Set(float64(score.Precision)) MemoryInUseBytesVec.WithLabelValues("collaborative_filtering_model").Set(float64(t.RankingModel.Bytes())) - if err := t.CacheClient.Set(cache.Time(cache.Key(cache.GlobalMeta, cache.LastFitMatchingModelTime), time.Now())); err != nil { + if err := t.CacheClient.Set(ctx, cache.Time(cache.Key(cache.GlobalMeta, cache.LastFitMatchingModelTime), time.Now())); err != nil { log.Logger().Error("failed to write meta", zap.Error(err)) } @@ -1075,6 +1090,7 @@ func (t *FitClickModelTask) run(j *task.JobsAllocator) error { numItems := t.clickTrainSet.ItemCount() numFeedback := t.clickTrainSet.Count() var shouldFit bool + ctx := context.Background() if t.clickTrainSet == nil || numUsers == 0 || numItems == 0 || numFeedback == 0 { log.Logger().Warn("empty ranking dataset", @@ -1129,7 +1145,7 @@ func (t *FitClickModelTask) run(j *task.JobsAllocator) error { RankingRecall.Set(float64(score.Recall)) RankingAUC.Set(float64(score.AUC)) MemoryInUseBytesVec.WithLabelValues("ranking_model").Set(float64(t.ClickModel.Bytes())) - if err := t.CacheClient.Set(cache.Time(cache.Key(cache.GlobalMeta, cache.LastFitRankingModelTime), time.Now())); err != nil { + if err := t.CacheClient.Set(ctx, cache.Time(cache.Key(cache.GlobalMeta, cache.LastFitRankingModelTime), time.Now())); err != nil { log.Logger().Error("failed to write meta", zap.Error(err)) } @@ -1304,6 +1320,7 @@ func (t *CacheGarbageCollectionTask) run(j *task.JobsAllocator) error { log.Logger().Debug("dataset has not been loaded") return nil } + ctx := context.Background() log.Logger().Info("start cache garbage collection") t.taskMonitor.Start(TaskCacheGarbageCollection, t.rankingTrainSet.UserCount()*9+t.rankingTrainSet.ItemCount()*4) @@ -1326,7 +1343,7 @@ func (t *CacheGarbageCollectionTask) run(j *task.JobsAllocator) error { return nil } // check user in database - _, err := t.DataClient.GetUser(userId) + _, err := t.DataClient.GetUser(ctx, userId) if !errors.Is(err, errors.NotFound) { if err != nil { log.Logger().Error("failed to load user", zap.String("user_id", userId), zap.Error(err)) @@ -1336,10 +1353,10 @@ func (t *CacheGarbageCollectionTask) run(j *task.JobsAllocator) error { // delete user cache switch splits[0] { case cache.UserNeighbors, cache.IgnoreItems, cache.CollaborativeRecommend, cache.OfflineRecommend: - err = t.CacheClient.SetSorted(s, nil) + err = t.CacheClient.SetSorted(ctx, s, nil) case cache.UserNeighborsDigest, cache.OfflineRecommendDigest, cache.LastModifyUserTime, cache.LastUpdateUserNeighborsTime, cache.LastUpdateUserRecommendTime: - err = t.CacheClient.Delete(s) + err = t.CacheClient.Delete(ctx, s) } if err != nil { return errors.Trace(err) @@ -1352,7 +1369,7 @@ func (t *CacheGarbageCollectionTask) run(j *task.JobsAllocator) error { return nil } // check item in database - _, err := t.DataClient.GetItem(itemId) + _, err := t.DataClient.GetItem(ctx, itemId) if !errors.Is(err, errors.NotFound) { if err != nil { log.Logger().Error("failed to load item", zap.String("item_id", itemId), zap.Error(err)) @@ -1362,9 +1379,9 @@ func (t *CacheGarbageCollectionTask) run(j *task.JobsAllocator) error { // delete item cache switch splits[0] { case cache.ItemNeighbors: - err = t.CacheClient.SetSorted(s, nil) + err = t.CacheClient.SetSorted(ctx, s, nil) case cache.ItemNeighborsDigest, cache.LastModifyItemTime, cache.LastUpdateItemNeighborsTime: - err = t.CacheClient.Delete(s) + err = t.CacheClient.Delete(ctx, s) } if err != nil { return errors.Trace(err) @@ -1374,7 +1391,7 @@ func (t *CacheGarbageCollectionTask) run(j *task.JobsAllocator) error { return nil }) // remove stale hidden items - if err := t.CacheClient.RemSortedByScore(cache.HiddenItemsV2, math.Inf(-1), float64(time.Now().Add(-t.Config.Recommend.CacheExpire).Unix())); err != nil { + if err := t.CacheClient.RemSortedByScore(ctx, cache.HiddenItemsV2, math.Inf(-1), float64(time.Now().Add(-t.Config.Recommend.CacheExpire).Unix())); err != nil { return errors.Trace(err) } t.taskMonitor.Finish(TaskCacheGarbageCollection) @@ -1388,7 +1405,7 @@ func (t *CacheGarbageCollectionTask) run(j *task.JobsAllocator) error { func (m *Master) LoadDataFromDatabase(database data.Database, posFeedbackTypes, readTypes []string, itemTTL, positiveFeedbackTTL uint, evaluator *OnlineEvaluator) ( rankingDataset *ranking.DataSet, clickDataset *click.Dataset, latestItems map[string][]cache.Scored, popularItems map[string][]cache.Scored, err error) { m.taskMonitor.Start(TaskLoadDataset, 5) - + ctx := context.Background() // setup time limit var itemTimeLimit, feedbackTimeLimit *time.Time if itemTTL > 0 { @@ -1414,7 +1431,7 @@ func (m *Master) LoadDataFromDatabase(database data.Database, posFeedbackTypes, userLabelFirst := make(map[string]int32) userLabelIndex := base.NewMapIndex() start := time.Now() - userChan, errChan := database.GetUserStream(batchSize) + userChan, errChan := database.GetUserStream(ctx, batchSize) for users := range userChan { for _, user := range users { rankingDataset.AddUser(user.UserId) @@ -1459,7 +1476,7 @@ func (m *Master) LoadDataFromDatabase(database data.Database, posFeedbackTypes, itemLabelFirst := make(map[string]int32) itemLabelIndex := base.NewMapIndex() start = time.Now() - itemChan, errChan := database.GetItemStream(batchSize, itemTimeLimit) + itemChan, errChan := database.GetItemStream(ctx, batchSize, itemTimeLimit) for items := range itemChan { for _, item := range items { rankingDataset.AddItem(item.ItemId) @@ -1523,7 +1540,7 @@ func (m *Master) LoadDataFromDatabase(database data.Database, posFeedbackTypes, // STEP 3: pull positive feedback var feedbackCount float64 start = time.Now() - feedbackChan, errChan := database.GetFeedbackStream(batchSize, feedbackTimeLimit, m.Config.Now(), posFeedbackTypes...) + feedbackChan, errChan := database.GetFeedbackStream(ctx, batchSize, feedbackTimeLimit, m.Config.Now(), posFeedbackTypes...) for feedback := range feedbackChan { for _, f := range feedback { feedbackCount++ @@ -1562,7 +1579,7 @@ func (m *Master) LoadDataFromDatabase(database data.Database, posFeedbackTypes, // STEP 4: pull negative feedback start = time.Now() - feedbackChan, errChan = database.GetFeedbackStream(batchSize, feedbackTimeLimit, m.Config.Now(), readTypes...) + feedbackChan, errChan = database.GetFeedbackStream(ctx, batchSize, feedbackTimeLimit, m.Config.Now(), readTypes...) for feedback := range feedbackChan { for _, f := range feedback { feedbackCount++ diff --git a/master/tasks_test.go b/master/tasks_test.go index 44e6d4c10..b7674972e 100644 --- a/master/tasks_test.go +++ b/master/tasks_test.go @@ -15,6 +15,7 @@ package master import ( + "context" "strconv" "testing" "time" @@ -31,6 +32,7 @@ func TestMaster_FindItemNeighborsBruteForce(t *testing.T) { // create mock master m := newMockMaster(t) defer m.Close() + ctx := context.Background() // create config m.Config = &config.Config{} m.Config.Recommend.CacheSize = 3 @@ -64,20 +66,20 @@ func TestMaster_FindItemNeighborsBruteForce(t *testing.T) { } } var err error - err = m.DataClient.BatchInsertItems(items) + err = m.DataClient.BatchInsertItems(ctx, items) assert.NoError(t, err) - err = m.DataClient.BatchInsertFeedback(feedbacks, true, true, true) + err = m.DataClient.BatchInsertFeedback(ctx, feedbacks, true, true, true) assert.NoError(t, err) // insert hidden item - err = m.DataClient.BatchInsertItems([]data.Item{{ + err = m.DataClient.BatchInsertItems(ctx, []data.Item{{ ItemId: "10", Labels: []string{"a", "b", "c", "d", "e"}, IsHidden: true, }}) assert.NoError(t, err) for i := 0; i <= 10; i++ { - err = m.DataClient.BatchInsertFeedback([]data.Feedback{{ + err = m.DataClient.BatchInsertFeedback(ctx, []data.Feedback{{ FeedbackKey: data.FeedbackKey{UserId: strconv.Itoa(i), ItemId: "10", FeedbackType: "FeedbackType"}, }}, true, true, true) assert.NoError(t, err) @@ -92,44 +94,44 @@ func TestMaster_FindItemNeighborsBruteForce(t *testing.T) { m.Config.Recommend.ItemNeighbors.NeighborType = config.NeighborTypeRelated neighborTask := NewFindItemNeighborsTask(&m.Master) assert.NoError(t, neighborTask.run(nil)) - similar, err := m.CacheClient.GetSorted(cache.Key(cache.ItemNeighbors, "9"), 0, 100) + similar, err := m.CacheClient.GetSorted(ctx, cache.Key(cache.ItemNeighbors, "9"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"7", "5", "3"}, cache.RemoveScores(similar)) assert.Equal(t, m.estimateFindItemNeighborsComplexity(dataset), m.taskMonitor.Tasks[TaskFindItemNeighbors].Done) assert.Equal(t, task.StatusComplete, m.taskMonitor.Tasks[TaskFindItemNeighbors].Status) // similar items in category (common users) - similar, err = m.CacheClient.GetSorted(cache.Key(cache.ItemNeighbors, "9", "*"), 0, 100) + similar, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.ItemNeighbors, "9", "*"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"7", "5", "1"}, cache.RemoveScores(similar)) // similar items (common labels) - err = m.CacheClient.Set(cache.Time(cache.Key(cache.LastModifyItemTime, "8"), time.Now())) + err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastModifyItemTime, "8"), time.Now())) assert.NoError(t, err) m.Config.Recommend.ItemNeighbors.NeighborType = config.NeighborTypeSimilar neighborTask = NewFindItemNeighborsTask(&m.Master) assert.NoError(t, neighborTask.run(nil)) - similar, err = m.CacheClient.GetSorted(cache.Key(cache.ItemNeighbors, "8"), 0, 100) + similar, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.ItemNeighbors, "8"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"0", "2", "4"}, cache.RemoveScores(similar)) assert.Equal(t, m.estimateFindItemNeighborsComplexity(dataset), m.taskMonitor.Tasks[TaskFindItemNeighbors].Done) assert.Equal(t, task.StatusComplete, m.taskMonitor.Tasks[TaskFindItemNeighbors].Status) // similar items in category (common labels) - similar, err = m.CacheClient.GetSorted(cache.Key(cache.ItemNeighbors, "8", "*"), 0, 100) + similar, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.ItemNeighbors, "8", "*"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"0", "2", "6"}, cache.RemoveScores(similar)) // similar items (auto) - err = m.CacheClient.Set(cache.Time(cache.Key(cache.LastModifyItemTime, "8"), time.Now())) + err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastModifyItemTime, "8"), time.Now())) assert.NoError(t, err) - err = m.CacheClient.Set(cache.Time(cache.Key(cache.LastModifyItemTime, "9"), time.Now())) + err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastModifyItemTime, "9"), time.Now())) assert.NoError(t, err) m.Config.Recommend.ItemNeighbors.NeighborType = config.NeighborTypeAuto neighborTask = NewFindItemNeighborsTask(&m.Master) assert.NoError(t, neighborTask.run(nil)) - similar, err = m.CacheClient.GetSorted(cache.Key(cache.ItemNeighbors, "8"), 0, 100) + similar, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.ItemNeighbors, "8"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"0", "2", "4"}, cache.RemoveScores(similar)) - similar, err = m.CacheClient.GetSorted(cache.Key(cache.ItemNeighbors, "9"), 0, 100) + similar, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.ItemNeighbors, "9"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"7", "5", "3"}, cache.RemoveScores(similar)) assert.Equal(t, m.estimateFindItemNeighborsComplexity(dataset), m.taskMonitor.Tasks[TaskFindItemNeighbors].Done) @@ -140,6 +142,7 @@ func TestMaster_FindItemNeighborsIVF(t *testing.T) { // create mock master m := newMockMaster(t) defer m.Close() + ctx := context.Background() // create config m.Config = &config.Config{} m.Config.Recommend.CacheSize = 3 @@ -176,20 +179,20 @@ func TestMaster_FindItemNeighborsIVF(t *testing.T) { } } var err error - err = m.DataClient.BatchInsertItems(items) + err = m.DataClient.BatchInsertItems(ctx, items) assert.NoError(t, err) - err = m.DataClient.BatchInsertFeedback(feedbacks, true, true, true) + err = m.DataClient.BatchInsertFeedback(ctx, feedbacks, true, true, true) assert.NoError(t, err) // insert hidden item - err = m.DataClient.BatchInsertItems([]data.Item{{ + err = m.DataClient.BatchInsertItems(ctx, []data.Item{{ ItemId: "10", Labels: []string{"a", "b", "c", "d", "e"}, IsHidden: true, }}) assert.NoError(t, err) for i := 0; i <= 10; i++ { - err = m.DataClient.BatchInsertFeedback([]data.Feedback{{ + err = m.DataClient.BatchInsertFeedback(ctx, []data.Feedback{{ FeedbackKey: data.FeedbackKey{UserId: strconv.Itoa(i), ItemId: "10", FeedbackType: "FeedbackType"}, }}, true, true, true) assert.NoError(t, err) @@ -204,44 +207,44 @@ func TestMaster_FindItemNeighborsIVF(t *testing.T) { m.Config.Recommend.ItemNeighbors.NeighborType = config.NeighborTypeRelated neighborTask := NewFindItemNeighborsTask(&m.Master) assert.NoError(t, neighborTask.run(nil)) - similar, err := m.CacheClient.GetSorted(cache.Key(cache.ItemNeighbors, "9"), 0, 100) + similar, err := m.CacheClient.GetSorted(ctx, cache.Key(cache.ItemNeighbors, "9"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"7", "5", "3"}, cache.RemoveScores(similar)) assert.Equal(t, m.estimateFindItemNeighborsComplexity(dataset), m.taskMonitor.Tasks[TaskFindItemNeighbors].Done) assert.Equal(t, task.StatusComplete, m.taskMonitor.Tasks[TaskFindItemNeighbors].Status) // similar items in category (common users) - similar, err = m.CacheClient.GetSorted(cache.Key(cache.ItemNeighbors, "9", "*"), 0, 100) + similar, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.ItemNeighbors, "9", "*"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"7", "5", "1"}, cache.RemoveScores(similar)) // similar items (common labels) - err = m.CacheClient.Set(cache.Time(cache.Key(cache.LastModifyItemTime, "8"), time.Now())) + err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastModifyItemTime, "8"), time.Now())) assert.NoError(t, err) m.Config.Recommend.ItemNeighbors.NeighborType = config.NeighborTypeSimilar neighborTask = NewFindItemNeighborsTask(&m.Master) assert.NoError(t, neighborTask.run(nil)) - similar, err = m.CacheClient.GetSorted(cache.Key(cache.ItemNeighbors, "8"), 0, 100) + similar, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.ItemNeighbors, "8"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"0", "2", "4"}, cache.RemoveScores(similar)) assert.Equal(t, m.estimateFindItemNeighborsComplexity(dataset), m.taskMonitor.Tasks[TaskFindItemNeighbors].Done) assert.Equal(t, task.StatusComplete, m.taskMonitor.Tasks[TaskFindItemNeighbors].Status) // similar items in category (common labels) - similar, err = m.CacheClient.GetSorted(cache.Key(cache.ItemNeighbors, "8", "*"), 0, 100) + similar, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.ItemNeighbors, "8", "*"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"0", "2", "6"}, cache.RemoveScores(similar)) // similar items (auto) - err = m.CacheClient.Set(cache.Time(cache.Key(cache.LastModifyItemTime, "8"), time.Now())) + err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastModifyItemTime, "8"), time.Now())) assert.NoError(t, err) - err = m.CacheClient.Set(cache.Time(cache.Key(cache.LastModifyItemTime, "9"), time.Now())) + err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastModifyItemTime, "9"), time.Now())) assert.NoError(t, err) m.Config.Recommend.ItemNeighbors.NeighborType = config.NeighborTypeAuto neighborTask = NewFindItemNeighborsTask(&m.Master) assert.NoError(t, neighborTask.run(nil)) - similar, err = m.CacheClient.GetSorted(cache.Key(cache.ItemNeighbors, "8"), 0, 100) + similar, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.ItemNeighbors, "8"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"0", "2", "4"}, cache.RemoveScores(similar)) - similar, err = m.CacheClient.GetSorted(cache.Key(cache.ItemNeighbors, "9"), 0, 100) + similar, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.ItemNeighbors, "9"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"7", "5", "3"}, cache.RemoveScores(similar)) assert.Equal(t, m.estimateFindItemNeighborsComplexity(dataset), m.taskMonitor.Tasks[TaskFindItemNeighbors].Done) @@ -252,6 +255,7 @@ func TestMaster_FindItemNeighborsIVF_ZeroIDF(t *testing.T) { // create mock master m := newMockMaster(t) defer m.Close() + ctx := context.Background() // create config m.Config = &config.Config{} m.Config.Recommend.CacheSize = 3 @@ -261,12 +265,12 @@ func TestMaster_FindItemNeighborsIVF_ZeroIDF(t *testing.T) { m.Config.Recommend.ItemNeighbors.IndexFitEpoch = 10 // create dataset - err := m.DataClient.BatchInsertItems([]data.Item{ + err := m.DataClient.BatchInsertItems(ctx, []data.Item{ {"0", false, []string{"*"}, time.Now(), []string{"a"}, ""}, {"1", false, []string{"*"}, time.Now(), []string{"a"}, ""}, }) assert.NoError(t, err) - err = m.DataClient.BatchInsertFeedback([]data.Feedback{ + err = m.DataClient.BatchInsertFeedback(ctx, []data.Feedback{ {FeedbackKey: data.FeedbackKey{FeedbackType: "FeedbackType", UserId: "0", ItemId: "0"}}, {FeedbackKey: data.FeedbackKey{FeedbackType: "FeedbackType", UserId: "0", ItemId: "1"}}, }, true, true, true) @@ -279,7 +283,7 @@ func TestMaster_FindItemNeighborsIVF_ZeroIDF(t *testing.T) { m.Config.Recommend.ItemNeighbors.NeighborType = config.NeighborTypeRelated neighborTask := NewFindItemNeighborsTask(&m.Master) assert.NoError(t, neighborTask.run(nil)) - similar, err := m.CacheClient.GetSorted(cache.Key(cache.ItemNeighbors, "0"), 0, 100) + similar, err := m.CacheClient.GetSorted(ctx, cache.Key(cache.ItemNeighbors, "0"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"1"}, cache.RemoveScores(similar)) @@ -287,7 +291,7 @@ func TestMaster_FindItemNeighborsIVF_ZeroIDF(t *testing.T) { m.Config.Recommend.ItemNeighbors.NeighborType = config.NeighborTypeSimilar neighborTask = NewFindItemNeighborsTask(&m.Master) assert.NoError(t, neighborTask.run(nil)) - similar, err = m.CacheClient.GetSorted(cache.Key(cache.ItemNeighbors, "0"), 0, 100) + similar, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.ItemNeighbors, "0"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"1"}, cache.RemoveScores(similar)) } @@ -296,6 +300,7 @@ func TestMaster_FindUserNeighborsBruteForce(t *testing.T) { // create mock master m := newMockMaster(t) defer m.Close() + ctx := context.Background() // create config m.Config = &config.Config{} m.Config.Recommend.CacheSize = 3 @@ -329,9 +334,9 @@ func TestMaster_FindUserNeighborsBruteForce(t *testing.T) { } } var err error - err = m.DataClient.BatchInsertUsers(users) + err = m.DataClient.BatchInsertUsers(ctx, users) assert.NoError(t, err) - err = m.DataClient.BatchInsertFeedback(feedbacks, true, true, true) + err = m.DataClient.BatchInsertFeedback(ctx, feedbacks, true, true, true) assert.NoError(t, err) dataset, _, _, _, err := m.LoadDataFromDatabase(m.DataClient, []string{"FeedbackType"}, nil, 0, 0, NewOnlineEvaluator()) assert.NoError(t, err) @@ -341,36 +346,36 @@ func TestMaster_FindUserNeighborsBruteForce(t *testing.T) { m.Config.Recommend.UserNeighbors.NeighborType = config.NeighborTypeRelated neighborTask := NewFindUserNeighborsTask(&m.Master) assert.NoError(t, neighborTask.run(nil)) - similar, err := m.CacheClient.GetSorted(cache.Key(cache.UserNeighbors, "9"), 0, 100) + similar, err := m.CacheClient.GetSorted(ctx, cache.Key(cache.UserNeighbors, "9"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"7", "5", "3"}, cache.RemoveScores(similar)) assert.Equal(t, m.estimateFindUserNeighborsComplexity(dataset), m.taskMonitor.Tasks[TaskFindUserNeighbors].Done) assert.Equal(t, task.StatusComplete, m.taskMonitor.Tasks[TaskFindUserNeighbors].Status) // similar items (common labels) - err = m.CacheClient.Set(cache.Time(cache.Key(cache.LastModifyUserTime, "8"), time.Now())) + err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastModifyUserTime, "8"), time.Now())) assert.NoError(t, err) m.Config.Recommend.UserNeighbors.NeighborType = config.NeighborTypeSimilar neighborTask = NewFindUserNeighborsTask(&m.Master) assert.NoError(t, neighborTask.run(nil)) - similar, err = m.CacheClient.GetSorted(cache.Key(cache.UserNeighbors, "8"), 0, 100) + similar, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.UserNeighbors, "8"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"0", "2", "4"}, cache.RemoveScores(similar)) assert.Equal(t, m.estimateFindUserNeighborsComplexity(dataset), m.taskMonitor.Tasks[TaskFindUserNeighbors].Done) assert.Equal(t, task.StatusComplete, m.taskMonitor.Tasks[TaskFindUserNeighbors].Status) // similar items (auto) - err = m.CacheClient.Set(cache.Time(cache.Key(cache.LastModifyUserTime, "8"), time.Now())) + err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastModifyUserTime, "8"), time.Now())) assert.NoError(t, err) - err = m.CacheClient.Set(cache.Time(cache.Key(cache.LastModifyUserTime, "9"), time.Now())) + err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastModifyUserTime, "9"), time.Now())) assert.NoError(t, err) m.Config.Recommend.UserNeighbors.NeighborType = config.NeighborTypeAuto neighborTask = NewFindUserNeighborsTask(&m.Master) assert.NoError(t, neighborTask.run(nil)) - similar, err = m.CacheClient.GetSorted(cache.Key(cache.UserNeighbors, "8"), 0, 100) + similar, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.UserNeighbors, "8"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"0", "2", "4"}, cache.RemoveScores(similar)) - similar, err = m.CacheClient.GetSorted(cache.Key(cache.UserNeighbors, "9"), 0, 100) + similar, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.UserNeighbors, "9"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"7", "5", "3"}, cache.RemoveScores(similar)) assert.Equal(t, m.estimateFindUserNeighborsComplexity(dataset), m.taskMonitor.Tasks[TaskFindUserNeighbors].Done) @@ -381,6 +386,7 @@ func TestMaster_FindUserNeighborsIVF(t *testing.T) { // create mock master m := newMockMaster(t) defer m.Close() + ctx := context.Background() // create config m.Config = &config.Config{} m.Config.Recommend.CacheSize = 3 @@ -417,9 +423,9 @@ func TestMaster_FindUserNeighborsIVF(t *testing.T) { } } var err error - err = m.DataClient.BatchInsertUsers(users) + err = m.DataClient.BatchInsertUsers(ctx, users) assert.NoError(t, err) - err = m.DataClient.BatchInsertFeedback(feedbacks, true, true, true) + err = m.DataClient.BatchInsertFeedback(ctx, feedbacks, true, true, true) assert.NoError(t, err) dataset, _, _, _, err := m.LoadDataFromDatabase(m.DataClient, []string{"FeedbackType"}, nil, 0, 0, NewOnlineEvaluator()) assert.NoError(t, err) @@ -429,36 +435,36 @@ func TestMaster_FindUserNeighborsIVF(t *testing.T) { m.Config.Recommend.UserNeighbors.NeighborType = config.NeighborTypeRelated neighborTask := NewFindUserNeighborsTask(&m.Master) assert.NoError(t, neighborTask.run(nil)) - similar, err := m.CacheClient.GetSorted(cache.Key(cache.UserNeighbors, "9"), 0, 100) + similar, err := m.CacheClient.GetSorted(ctx, cache.Key(cache.UserNeighbors, "9"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"7", "5", "3"}, cache.RemoveScores(similar)) assert.Equal(t, m.estimateFindUserNeighborsComplexity(dataset), m.taskMonitor.Tasks[TaskFindUserNeighbors].Done) assert.Equal(t, task.StatusComplete, m.taskMonitor.Tasks[TaskFindUserNeighbors].Status) // similar items (common labels) - err = m.CacheClient.Set(cache.Time(cache.Key(cache.LastModifyUserTime, "8"), time.Now())) + err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastModifyUserTime, "8"), time.Now())) assert.NoError(t, err) m.Config.Recommend.UserNeighbors.NeighborType = config.NeighborTypeSimilar neighborTask = NewFindUserNeighborsTask(&m.Master) assert.NoError(t, neighborTask.run(nil)) - similar, err = m.CacheClient.GetSorted(cache.Key(cache.UserNeighbors, "8"), 0, 100) + similar, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.UserNeighbors, "8"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"0", "2", "4"}, cache.RemoveScores(similar)) assert.Equal(t, m.estimateFindUserNeighborsComplexity(dataset), m.taskMonitor.Tasks[TaskFindUserNeighbors].Done) assert.Equal(t, task.StatusComplete, m.taskMonitor.Tasks[TaskFindUserNeighbors].Status) // similar items (auto) - err = m.CacheClient.Set(cache.Time(cache.Key(cache.LastModifyUserTime, "8"), time.Now())) + err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastModifyUserTime, "8"), time.Now())) assert.NoError(t, err) - err = m.CacheClient.Set(cache.Time(cache.Key(cache.LastModifyUserTime, "9"), time.Now())) + err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastModifyUserTime, "9"), time.Now())) assert.NoError(t, err) m.Config.Recommend.UserNeighbors.NeighborType = config.NeighborTypeAuto neighborTask = NewFindUserNeighborsTask(&m.Master) assert.NoError(t, neighborTask.run(nil)) - similar, err = m.CacheClient.GetSorted(cache.Key(cache.UserNeighbors, "8"), 0, 100) + similar, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.UserNeighbors, "8"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"0", "2", "4"}, cache.RemoveScores(similar)) - similar, err = m.CacheClient.GetSorted(cache.Key(cache.UserNeighbors, "9"), 0, 100) + similar, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.UserNeighbors, "9"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"7", "5", "3"}, cache.RemoveScores(similar)) assert.Equal(t, m.estimateFindUserNeighborsComplexity(dataset), m.taskMonitor.Tasks[TaskFindUserNeighbors].Done) @@ -469,6 +475,7 @@ func TestMaster_FindUserNeighborsIVF_ZeroIDF(t *testing.T) { // create mock master m := newMockMaster(t) defer m.Close() + ctx := context.Background() // create config m.Config = &config.Config{} m.Config.Recommend.CacheSize = 3 @@ -478,12 +485,12 @@ func TestMaster_FindUserNeighborsIVF_ZeroIDF(t *testing.T) { m.Config.Recommend.UserNeighbors.IndexFitEpoch = 10 // create dataset - err := m.DataClient.BatchInsertUsers([]data.User{ + err := m.DataClient.BatchInsertUsers(ctx, []data.User{ {"0", []string{"a"}, nil, ""}, {"1", []string{"a"}, nil, ""}, }) assert.NoError(t, err) - err = m.DataClient.BatchInsertFeedback([]data.Feedback{ + err = m.DataClient.BatchInsertFeedback(ctx, []data.Feedback{ {FeedbackKey: data.FeedbackKey{FeedbackType: "FeedbackType", UserId: "0", ItemId: "0"}}, {FeedbackKey: data.FeedbackKey{FeedbackType: "FeedbackType", UserId: "1", ItemId: "0"}}, }, true, true, true) @@ -496,7 +503,7 @@ func TestMaster_FindUserNeighborsIVF_ZeroIDF(t *testing.T) { m.Config.Recommend.UserNeighbors.NeighborType = config.NeighborTypeRelated neighborTask := NewFindUserNeighborsTask(&m.Master) assert.NoError(t, neighborTask.run(nil)) - similar, err := m.CacheClient.GetSorted(cache.Key(cache.UserNeighbors, "0"), 0, 100) + similar, err := m.CacheClient.GetSorted(ctx, cache.Key(cache.UserNeighbors, "0"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"1"}, cache.RemoveScores(similar)) @@ -504,7 +511,7 @@ func TestMaster_FindUserNeighborsIVF_ZeroIDF(t *testing.T) { m.Config.Recommend.UserNeighbors.NeighborType = config.NeighborTypeSimilar neighborTask = NewFindUserNeighborsTask(&m.Master) assert.NoError(t, neighborTask.run(nil)) - similar, err = m.CacheClient.GetSorted(cache.Key(cache.UserNeighbors, "0"), 0, 100) + similar, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.UserNeighbors, "0"), 0, 100) assert.NoError(t, err) assert.Equal(t, []string{"1"}, cache.RemoveScores(similar)) } @@ -513,6 +520,7 @@ func TestMaster_LoadDataFromDatabase(t *testing.T) { // create mock master m := newMockMaster(t) defer m.Close() + ctx := context.Background() // create config m.Config = &config.Config{} m.Config.Recommend.CacheSize = 3 @@ -529,9 +537,9 @@ func TestMaster_LoadDataFromDatabase(t *testing.T) { Categories: []string{strconv.Itoa(i % 3)}, }) } - err := m.DataClient.BatchInsertItems(items) + err := m.DataClient.BatchInsertItems(ctx, items) assert.NoError(t, err) - err = m.DataClient.BatchInsertItems([]data.Item{{ + err = m.DataClient.BatchInsertItems(ctx, []data.Item{{ ItemId: "9", Timestamp: time.Date(2020, 1, 1, 1, 1, 0, 0, time.UTC), IsHidden: true, @@ -546,7 +554,7 @@ func TestMaster_LoadDataFromDatabase(t *testing.T) { Labels: []string{strconv.Itoa(i % 5), strconv.Itoa(i*10 + 10)}, }) } - err = m.DataClient.BatchInsertUsers(users) + err = m.DataClient.BatchInsertUsers(ctx, users) assert.NoError(t, err) // insert feedback @@ -581,7 +589,7 @@ func TestMaster_LoadDataFromDatabase(t *testing.T) { }) } } - err = m.DataClient.BatchInsertFeedback(feedbacks, false, false, true) + err = m.DataClient.BatchInsertFeedback(ctx, feedbacks, false, false, true) assert.NoError(t, err) // load dataset @@ -605,14 +613,14 @@ func TestMaster_LoadDataFromDatabase(t *testing.T) { assert.Equal(t, 45, m.clickTrainSet.NegativeCount+m.clickTestSet.NegativeCount) // check latest items - latest, err := m.CacheClient.GetSorted(cache.Key(cache.LatestItems, ""), 0, 100) + latest, err := m.CacheClient.GetSorted(ctx, cache.Key(cache.LatestItems, ""), 0, 100) assert.NoError(t, err) assert.Equal(t, []cache.Scored{ {items[8].ItemId, float64(items[8].Timestamp.Unix())}, {items[7].ItemId, float64(items[7].Timestamp.Unix())}, {items[6].ItemId, float64(items[6].Timestamp.Unix())}, }, latest) - latest, err = m.CacheClient.GetSorted(cache.Key(cache.LatestItems, "2"), 0, 100) + latest, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.LatestItems, "2"), 0, 100) assert.NoError(t, err) assert.Equal(t, []cache.Scored{ {items[8].ItemId, float64(items[8].Timestamp.Unix())}, @@ -621,14 +629,14 @@ func TestMaster_LoadDataFromDatabase(t *testing.T) { }, latest) // check popular items - popular, err := m.CacheClient.GetSorted(cache.Key(cache.PopularItems, ""), 0, 2) + popular, err := m.CacheClient.GetSorted(ctx, cache.Key(cache.PopularItems, ""), 0, 2) assert.NoError(t, err) assert.Equal(t, []cache.Scored{ {Id: items[8].ItemId, Score: 9}, {Id: items[7].ItemId, Score: 8}, {Id: items[6].ItemId, Score: 7}, }, popular) - popular, err = m.CacheClient.GetSorted(cache.Key(cache.PopularItems, "2"), 0, 2) + popular, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.PopularItems, "2"), 0, 2) assert.NoError(t, err) assert.Equal(t, []cache.Scored{ {Id: items[8].ItemId, Score: 9}, @@ -637,7 +645,7 @@ func TestMaster_LoadDataFromDatabase(t *testing.T) { }, popular) // check categories - categories, err := m.CacheClient.GetSet(cache.ItemCategories) + categories, err := m.CacheClient.GetSet(ctx, cache.ItemCategories) assert.NoError(t, err) assert.Equal(t, []string{"0", "1", "2"}, categories) } @@ -647,10 +655,11 @@ func TestCheckItemNeighborCacheTimeout(t *testing.T) { m := newMockMaster(t) defer m.Close() m.Config = config.GetDefaultConfig() + ctx := context.Background() // empty cache assert.True(t, m.checkItemNeighborCacheTimeout("1", nil)) - err := m.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "1"), []cache.Scored{ + err := m.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "1"), []cache.Scored{ {Id: "2", Score: 1}, {Id: "3", Score: 2}, {Id: "4", Score: 3}, @@ -658,23 +667,23 @@ func TestCheckItemNeighborCacheTimeout(t *testing.T) { assert.NoError(t, err) // digest mismatch - err = m.CacheClient.Set(cache.String(cache.Key(cache.ItemNeighborsDigest, "1"), "digest")) + err = m.CacheClient.Set(ctx, cache.String(cache.Key(cache.ItemNeighborsDigest, "1"), "digest")) assert.NoError(t, err) assert.True(t, m.checkItemNeighborCacheTimeout("1", nil)) // staled cache - err = m.CacheClient.Set(cache.String(cache.Key(cache.ItemNeighborsDigest, "1"), m.Config.ItemNeighborDigest())) + err = m.CacheClient.Set(ctx, cache.String(cache.Key(cache.ItemNeighborsDigest, "1"), m.Config.ItemNeighborDigest())) assert.NoError(t, err) assert.True(t, m.checkItemNeighborCacheTimeout("1", nil)) - err = m.CacheClient.Set(cache.Time(cache.Key(cache.LastModifyItemTime, "1"), time.Now().Add(-time.Minute))) + err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastModifyItemTime, "1"), time.Now().Add(-time.Minute))) assert.NoError(t, err) assert.True(t, m.checkItemNeighborCacheTimeout("1", nil)) - err = m.CacheClient.Set(cache.Time(cache.Key(cache.LastUpdateItemNeighborsTime, "1"), time.Now().Add(-time.Hour))) + err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastUpdateItemNeighborsTime, "1"), time.Now().Add(-time.Hour))) assert.NoError(t, err) assert.True(t, m.checkItemNeighborCacheTimeout("1", nil)) // not staled cache - err = m.CacheClient.Set(cache.Time(cache.Key(cache.LastUpdateItemNeighborsTime, "1"), time.Now())) + err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastUpdateItemNeighborsTime, "1"), time.Now())) assert.NoError(t, err) assert.False(t, m.checkItemNeighborCacheTimeout("1", nil)) } @@ -683,11 +692,12 @@ func TestCheckUserNeighborCacheTimeout(t *testing.T) { // create mock master m := newMockMaster(t) defer m.Close() + ctx := context.Background() m.Config = config.GetDefaultConfig() // empty cache assert.True(t, m.checkUserNeighborCacheTimeout("1")) - err := m.CacheClient.SetSorted(cache.Key(cache.UserNeighbors, "1"), []cache.Scored{ + err := m.CacheClient.SetSorted(ctx, cache.Key(cache.UserNeighbors, "1"), []cache.Scored{ {Id: "1", Score: 1}, {Id: "2", Score: 2}, {Id: "3", Score: 3}, @@ -695,23 +705,23 @@ func TestCheckUserNeighborCacheTimeout(t *testing.T) { assert.NoError(t, err) // digest mismatch - err = m.CacheClient.Set(cache.String(cache.Key(cache.UserNeighborsDigest, "1"), "digest")) + err = m.CacheClient.Set(ctx, cache.String(cache.Key(cache.UserNeighborsDigest, "1"), "digest")) assert.NoError(t, err) assert.True(t, m.checkUserNeighborCacheTimeout("1")) // staled cache - err = m.CacheClient.Set(cache.String(cache.Key(cache.UserNeighborsDigest, "1"), m.Config.UserNeighborDigest())) + err = m.CacheClient.Set(ctx, cache.String(cache.Key(cache.UserNeighborsDigest, "1"), m.Config.UserNeighborDigest())) assert.NoError(t, err) assert.True(t, m.checkUserNeighborCacheTimeout("1")) - err = m.CacheClient.Set(cache.Time(cache.Key(cache.LastModifyUserTime, "1"), time.Now().Add(-time.Minute))) + err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastModifyUserTime, "1"), time.Now().Add(-time.Minute))) assert.NoError(t, err) assert.True(t, m.checkUserNeighborCacheTimeout("1")) - err = m.CacheClient.Set(cache.Time(cache.Key(cache.LastUpdateUserNeighborsTime, "1"), time.Now().Add(-time.Hour))) + err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastUpdateUserNeighborsTime, "1"), time.Now().Add(-time.Hour))) assert.NoError(t, err) assert.True(t, m.checkUserNeighborCacheTimeout("1")) // not staled cache - err = m.CacheClient.Set(cache.Time(cache.Key(cache.LastUpdateUserNeighborsTime, "1"), time.Now())) + err = m.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastUpdateUserNeighborsTime, "1"), time.Now())) assert.NoError(t, err) assert.False(t, m.checkUserNeighborCacheTimeout("1")) } @@ -721,16 +731,17 @@ func TestRunCacheGarbageCollectionTask(t *testing.T) { m := newMockMaster(t) defer m.Close() m.Config = config.GetDefaultConfig() + ctx := context.Background() // insert data - err := m.DataClient.BatchInsertFeedback([]data.Feedback{{FeedbackKey: data.FeedbackKey{UserId: "1", ItemId: "10"}}}, true, true, true) + err := m.DataClient.BatchInsertFeedback(ctx, []data.Feedback{{FeedbackKey: data.FeedbackKey{UserId: "1", ItemId: "10"}}}, true, true, true) assert.NoError(t, err) err = m.runLoadDatasetTask() assert.NoError(t, err) // insert cache timestamp := time.Now() - err = m.CacheClient.Set( + err = m.CacheClient.Set(ctx, cache.String(cache.Key(cache.UserNeighborsDigest, "1"), "digest"), cache.String(cache.Key(cache.OfflineRecommendDigest, "1"), "digest"), cache.Time(cache.Key(cache.LastModifyUserTime, "1"), timestamp), @@ -738,22 +749,22 @@ func TestRunCacheGarbageCollectionTask(t *testing.T) { cache.Time(cache.Key(cache.LastUpdateUserRecommendTime, "1"), timestamp), ) assert.NoError(t, err) - err = m.CacheClient.SetSorted(cache.Key(cache.UserNeighbors, "1"), []cache.Scored{{Id: "1", Score: 1}}) + err = m.CacheClient.SetSorted(ctx, cache.Key(cache.UserNeighbors, "1"), []cache.Scored{{Id: "1", Score: 1}}) assert.NoError(t, err) - err = m.CacheClient.SetSorted(cache.Key(cache.CollaborativeRecommend, "1"), []cache.Scored{{Id: "1", Score: 1}}) + err = m.CacheClient.SetSorted(ctx, cache.Key(cache.CollaborativeRecommend, "1"), []cache.Scored{{Id: "1", Score: 1}}) assert.NoError(t, err) - err = m.CacheClient.SetSorted(cache.Key(cache.OfflineRecommend, "1"), []cache.Scored{{Id: "1", Score: 1}}) + err = m.CacheClient.SetSorted(ctx, cache.Key(cache.OfflineRecommend, "1"), []cache.Scored{{Id: "1", Score: 1}}) assert.NoError(t, err) - err = m.CacheClient.Set( + err = m.CacheClient.Set(ctx, cache.String(cache.Key(cache.ItemNeighborsDigest, "10"), "digest"), cache.Time(cache.Key(cache.LastModifyItemTime, "10"), timestamp), cache.Time(cache.Key(cache.LastUpdateItemNeighborsTime, "10"), timestamp), ) assert.NoError(t, err) - err = m.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "10"), []cache.Scored{{Id: "1", Score: 1}}) + err = m.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "10"), []cache.Scored{{Id: "1", Score: 1}}) assert.NoError(t, err) - err = m.CacheClient.Set( + err = m.CacheClient.Set(ctx, cache.String(cache.Key(cache.UserNeighborsDigest, "2"), "digest"), cache.String(cache.Key(cache.OfflineRecommendDigest, "2"), "digest"), cache.Time(cache.Key(cache.LastModifyUserTime, "2"), timestamp), @@ -761,19 +772,19 @@ func TestRunCacheGarbageCollectionTask(t *testing.T) { cache.Time(cache.Key(cache.LastUpdateUserRecommendTime, "2"), timestamp), ) assert.NoError(t, err) - err = m.CacheClient.SetSorted(cache.Key(cache.UserNeighbors, "2"), []cache.Scored{{Id: "1", Score: 1}}) + err = m.CacheClient.SetSorted(ctx, cache.Key(cache.UserNeighbors, "2"), []cache.Scored{{Id: "1", Score: 1}}) assert.NoError(t, err) - err = m.CacheClient.SetSorted(cache.Key(cache.CollaborativeRecommend, "2"), []cache.Scored{{Id: "1", Score: 1}}) + err = m.CacheClient.SetSorted(ctx, cache.Key(cache.CollaborativeRecommend, "2"), []cache.Scored{{Id: "1", Score: 1}}) assert.NoError(t, err) - err = m.CacheClient.SetSorted(cache.Key(cache.OfflineRecommend, "2"), []cache.Scored{{Id: "1", Score: 1}}) + err = m.CacheClient.SetSorted(ctx, cache.Key(cache.OfflineRecommend, "2"), []cache.Scored{{Id: "1", Score: 1}}) assert.NoError(t, err) - err = m.CacheClient.Set( + err = m.CacheClient.Set(ctx, cache.String(cache.Key(cache.ItemNeighborsDigest, "20"), "digest"), cache.Time(cache.Key(cache.LastModifyItemTime, "20"), timestamp), cache.Time(cache.Key(cache.LastUpdateItemNeighborsTime, "20"), timestamp), ) assert.NoError(t, err) - err = m.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "20"), []cache.Scored{{Id: "2", Score: 1}}) + err = m.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "20"), []cache.Scored{{Id: "2", Score: 1}}) assert.NoError(t, err) // remove cache @@ -783,72 +794,72 @@ func TestRunCacheGarbageCollectionTask(t *testing.T) { assert.NoError(t, err) var s string - s, err = m.CacheClient.Get(cache.Key(cache.UserNeighborsDigest, "1")).String() + s, err = m.CacheClient.Get(ctx, cache.Key(cache.UserNeighborsDigest, "1")).String() assert.NoError(t, err) assert.Equal(t, "digest", s) - s, err = m.CacheClient.Get(cache.Key(cache.OfflineRecommendDigest, "1")).String() + s, err = m.CacheClient.Get(ctx, cache.Key(cache.OfflineRecommendDigest, "1")).String() assert.NoError(t, err) assert.Equal(t, "digest", s) var ts time.Time - ts, err = m.CacheClient.Get(cache.Key(cache.LastModifyUserTime, "1")).Time() + ts, err = m.CacheClient.Get(ctx, cache.Key(cache.LastModifyUserTime, "1")).Time() assert.NoError(t, err) assert.Equal(t, timestamp.Truncate(time.Second), ts.Truncate(time.Second)) - ts, err = m.CacheClient.Get(cache.Key(cache.LastUpdateUserNeighborsTime, "1")).Time() + ts, err = m.CacheClient.Get(ctx, cache.Key(cache.LastUpdateUserNeighborsTime, "1")).Time() assert.NoError(t, err) assert.Equal(t, timestamp.Truncate(time.Second), ts.Truncate(time.Second)) - ts, err = m.CacheClient.Get(cache.Key(cache.LastUpdateUserRecommendTime, "1")).Time() + ts, err = m.CacheClient.Get(ctx, cache.Key(cache.LastUpdateUserRecommendTime, "1")).Time() assert.NoError(t, err) assert.Equal(t, timestamp.Truncate(time.Second), ts.Truncate(time.Second)) - sorted, err := m.CacheClient.GetSorted(cache.Key(cache.UserNeighbors, "1"), 0, -1) + sorted, err := m.CacheClient.GetSorted(ctx, cache.Key(cache.UserNeighbors, "1"), 0, -1) assert.NoError(t, err) assert.Equal(t, []cache.Scored{{Id: "1", Score: 1}}, sorted) - sorted, err = m.CacheClient.GetSorted(cache.Key(cache.CollaborativeRecommend, "1"), 0, -1) + sorted, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.CollaborativeRecommend, "1"), 0, -1) assert.NoError(t, err) assert.Equal(t, []cache.Scored{{Id: "1", Score: 1}}, sorted) - sorted, err = m.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, "1"), 0, -1) + sorted, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.OfflineRecommend, "1"), 0, -1) assert.NoError(t, err) assert.Equal(t, []cache.Scored{{Id: "1", Score: 1}}, sorted) - s, err = m.CacheClient.Get(cache.Key(cache.ItemNeighborsDigest, "10")).String() + s, err = m.CacheClient.Get(ctx, cache.Key(cache.ItemNeighborsDigest, "10")).String() assert.NoError(t, err) assert.Equal(t, "digest", s) - ts, err = m.CacheClient.Get(cache.Key(cache.LastModifyItemTime, "10")).Time() + ts, err = m.CacheClient.Get(ctx, cache.Key(cache.LastModifyItemTime, "10")).Time() assert.NoError(t, err) assert.Equal(t, timestamp.Truncate(time.Second), ts.Truncate(time.Second)) - ts, err = m.CacheClient.Get(cache.Key(cache.LastUpdateItemNeighborsTime, "10")).Time() + ts, err = m.CacheClient.Get(ctx, cache.Key(cache.LastUpdateItemNeighborsTime, "10")).Time() assert.NoError(t, err) assert.Equal(t, timestamp.Truncate(time.Second), ts.Truncate(time.Second)) - sorted, err = m.CacheClient.GetSorted(cache.Key(cache.ItemNeighbors, "10"), 0, -1) + sorted, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.ItemNeighbors, "10"), 0, -1) assert.NoError(t, err) assert.Equal(t, []cache.Scored{{Id: "1", Score: 1}}, sorted) - _, err = m.CacheClient.Get(cache.Key(cache.UserNeighborsDigest, "2")).String() + _, err = m.CacheClient.Get(ctx, cache.Key(cache.UserNeighborsDigest, "2")).String() assert.True(t, errors.Is(err, errors.NotFound)) - _, err = m.CacheClient.Get(cache.Key(cache.OfflineRecommendDigest, "2")).String() + _, err = m.CacheClient.Get(ctx, cache.Key(cache.OfflineRecommendDigest, "2")).String() assert.True(t, errors.Is(err, errors.NotFound)) - _, err = m.CacheClient.Get(cache.Key(cache.LastModifyUserTime, "2")).Time() + _, err = m.CacheClient.Get(ctx, cache.Key(cache.LastModifyUserTime, "2")).Time() assert.True(t, errors.Is(err, errors.NotFound)) - _, err = m.CacheClient.Get(cache.Key(cache.LastUpdateUserNeighborsTime, "2")).Time() + _, err = m.CacheClient.Get(ctx, cache.Key(cache.LastUpdateUserNeighborsTime, "2")).Time() assert.True(t, errors.Is(err, errors.NotFound)) - _, err = m.CacheClient.Get(cache.Key(cache.LastUpdateUserRecommendTime, "2")).Time() + _, err = m.CacheClient.Get(ctx, cache.Key(cache.LastUpdateUserRecommendTime, "2")).Time() assert.True(t, errors.Is(err, errors.NotFound)) - sorted, err = m.CacheClient.GetSorted(cache.Key(cache.UserNeighbors, "2"), 0, -1) + sorted, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.UserNeighbors, "2"), 0, -1) assert.NoError(t, err) assert.Empty(t, sorted) - sorted, err = m.CacheClient.GetSorted(cache.Key(cache.CollaborativeRecommend, "2"), 0, -1) + sorted, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.CollaborativeRecommend, "2"), 0, -1) assert.NoError(t, err) assert.Empty(t, sorted) - sorted, err = m.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, "2"), 0, -1) + sorted, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.OfflineRecommend, "2"), 0, -1) assert.NoError(t, err) assert.Empty(t, sorted) - _, err = m.CacheClient.Get(cache.Key(cache.ItemNeighborsDigest, "20")).String() + _, err = m.CacheClient.Get(ctx, cache.Key(cache.ItemNeighborsDigest, "20")).String() assert.True(t, errors.Is(err, errors.NotFound)) - _, err = m.CacheClient.Get(cache.Key(cache.LastModifyItemTime, "20")).Time() + _, err = m.CacheClient.Get(ctx, cache.Key(cache.LastModifyItemTime, "20")).Time() assert.True(t, errors.Is(err, errors.NotFound)) - _, err = m.CacheClient.Get(cache.Key(cache.LastUpdateItemNeighborsTime, "20")).Time() + _, err = m.CacheClient.Get(ctx, cache.Key(cache.LastUpdateItemNeighborsTime, "20")).Time() assert.True(t, errors.Is(err, errors.NotFound)) - sorted, err = m.CacheClient.GetSorted(cache.Key(cache.ItemNeighbors, "20"), 0, -1) + sorted, err = m.CacheClient.GetSorted(ctx, cache.Key(cache.ItemNeighbors, "20"), 0, -1) assert.NoError(t, err) assert.Empty(t, sorted) } diff --git a/server/bench_test.go b/server/bench_test.go index 75b8a888c..3604ee352 100644 --- a/server/bench_test.go +++ b/server/bench_test.go @@ -71,6 +71,7 @@ type benchServer struct { } func newBenchServer(b *testing.B) *benchServer { + ctx := context.Background() // retrieve benchmark name var benchName string pc, _, _, ok := runtime.Caller(1) @@ -116,7 +117,7 @@ func newBenchServer(b *testing.B) *benchServer { Comment: fmt.Sprintf("comment_%d", i), }) } - err = s.DataClient.BatchInsertUsers(users) + err = s.DataClient.BatchInsertUsers(ctx, users) require.NoError(b, err) // insert items @@ -137,7 +138,7 @@ func newBenchServer(b *testing.B) *benchServer { IsHidden: false, }) } - err = s.DataClient.BatchInsertItems(items) + err = s.DataClient.BatchInsertItems(ctx, items) require.NoError(b, err) // insert feedback @@ -153,7 +154,7 @@ func newBenchServer(b *testing.B) *benchServer { Timestamp: time.Now(), }) } - err = s.DataClient.BatchInsertFeedback(feedbacks, true, true, true) + err = s.DataClient.BatchInsertFeedback(ctx, feedbacks, true, true, true) require.NoError(b, err) // start http server @@ -792,6 +793,7 @@ func BenchmarkGetRecommendCache(b *testing.B) { s := newBenchServer(b) defer s.Close(b) + ctx := context.Background() for batchSize := 10; batchSize <= 1000; batchSize *= 10 { b.Run(strconv.Itoa(batchSize), func(b *testing.B) { scores := make([]cache.Scored, batchSize*2) @@ -808,7 +810,7 @@ func BenchmarkGetRecommendCache(b *testing.B) { } lo.Reverse(scores) lo.Reverse(expects) - err := s.CacheClient.SetSorted(cache.PopularItems, scores) + err := s.CacheClient.SetSorted(ctx, cache.PopularItems, scores) require.NoError(b, err) s.Config.Recommend.CacheSize = len(scores) @@ -837,6 +839,7 @@ func BenchmarkRecommendFromOfflineCache(b *testing.B) { s := newBenchServer(b) defer s.Close(b) + ctx := context.Background() for batchSize := 10; batchSize <= 1000; batchSize *= 10 { b.Run(strconv.Itoa(batchSize), func(b *testing.B) { scores := make([]cache.Scored, batchSize*2) @@ -847,13 +850,13 @@ func BenchmarkRecommendFromOfflineCache(b *testing.B) { if i%2 == 0 { expects[i/2] = scores[i].Id } else { - err := s.CacheClient.AddSorted(cache.Sorted(cache.Key(cache.IgnoreItems, "init_user_1"), []cache.Scored{{scores[i].Id, 0}})) + err := s.CacheClient.AddSorted(ctx, cache.Sorted(cache.Key(cache.IgnoreItems, "init_user_1"), []cache.Scored{{scores[i].Id, 0}})) require.NoError(b, err) } } lo.Reverse(scores) lo.Reverse(expects) - err := s.CacheClient.SetSorted(cache.Key(cache.OfflineRecommend, "init_user_1"), scores) + err := s.CacheClient.SetSorted(ctx, cache.Key(cache.OfflineRecommend, "init_user_1"), scores) require.NoError(b, err) s.Config.Recommend.CacheSize = len(scores) @@ -878,6 +881,7 @@ func BenchmarkRecommendFromOfflineCache(b *testing.B) { } func BenchmarkRecommendFromLatest(b *testing.B) { + ctx := context.Background() log.CloseLogger() s := newBenchServer(b) defer s.Close(b) @@ -892,7 +896,7 @@ func BenchmarkRecommendFromLatest(b *testing.B) { if i%2 == 0 { expects[i/2] = scores[i].Id } else { - err := s.DataClient.BatchInsertFeedback([]data.Feedback{{ + err := s.DataClient.BatchInsertFeedback(ctx, []data.Feedback{{ FeedbackKey: data.FeedbackKey{ FeedbackType: "feedback_type", UserId: "init_user_1", @@ -904,7 +908,7 @@ func BenchmarkRecommendFromLatest(b *testing.B) { } lo.Reverse(scores) lo.Reverse(expects) - err := s.CacheClient.SetSorted(cache.LatestItems, scores) + err := s.CacheClient.SetSorted(ctx, cache.LatestItems, scores) require.NoError(b, err) s.Config.Recommend.CacheSize = len(scores) @@ -929,6 +933,7 @@ func BenchmarkRecommendFromLatest(b *testing.B) { } func BenchmarkRecommendFromItemBased(b *testing.B) { + ctx := context.Background() log.CloseLogger() s := newBenchServer(b) defer s.Close(b) @@ -941,7 +946,7 @@ func BenchmarkRecommendFromItemBased(b *testing.B) { scores[i].Id = fmt.Sprintf("init_item_%d", i) scores[i].Score = float64(i) if i < s.Config.Recommend.Online.NumFeedbackFallbackItemBased { - err := s.DataClient.BatchInsertFeedback([]data.Feedback{{ + err := s.DataClient.BatchInsertFeedback(ctx, []data.Feedback{{ FeedbackKey: data.FeedbackKey{ FeedbackType: "feedback_type_positive", UserId: "init_user_1", @@ -955,7 +960,7 @@ func BenchmarkRecommendFromItemBased(b *testing.B) { // insert user neighbors for i := 0; i < s.Config.Recommend.Online.NumFeedbackFallbackItemBased; i++ { - err := s.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, fmt.Sprintf("init_item_%d", i)), scores) + err := s.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, fmt.Sprintf("init_item_%d", i)), scores) require.NoError(b, err) } diff --git a/server/rest.go b/server/rest.go index 761f3a04f..f53980fa9 100644 --- a/server/rest.go +++ b/server/rest.go @@ -15,6 +15,7 @@ package server import ( + "context" "encoding/json" "fmt" "math" @@ -38,6 +39,7 @@ import ( "github.com/zhenghaoz/gorse/config" "github.com/zhenghaoz/gorse/storage/cache" "github.com/zhenghaoz/gorse/storage/data" + "go.opentelemetry.io/contrib/instrumentation/github.com/emicklei/go-restful/otelrestful" "go.uber.org/zap" "modernc.org/mathutil" ) @@ -150,7 +152,8 @@ func (s *RestServer) CreateWebService() { Produces(restful.MIME_JSON). Filter(s.LogFilter). Filter(s.AuthFilter). - Filter(s.MetricsFilter) + Filter(s.MetricsFilter). + Filter(otelrestful.OTelFilter("gorse")) /* Interactions with data store */ @@ -550,8 +553,12 @@ func (s *RestServer) getSort(key, category string, isItem bool, request *restful BadRequest(response, err) return } + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } // Get the popular list - items, err := s.CacheClient.GetSorted(cache.Key(key, category), offset, s.Config.Recommend.CacheSize) + items, err := s.CacheClient.GetSorted(ctx, cache.Key(key, category), offset, s.Config.Recommend.CacheSize) if err != nil { InternalServerError(response, err) return @@ -580,9 +587,13 @@ func (s *RestServer) getLatest(request *restful.Request, response *restful.Respo // get feedback by item-id with feedback type func (s *RestServer) getTypedFeedbackByItem(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } feedbackType := request.PathParameter("feedback-type") itemId := request.PathParameter("item-id") - feedback, err := s.DataClient.GetItemFeedback(itemId, feedbackType) + feedback, err := s.DataClient.GetItemFeedback(ctx, itemId, feedbackType) if err != nil { InternalServerError(response, err) return @@ -592,8 +603,12 @@ func (s *RestServer) getTypedFeedbackByItem(request *restful.Request, response * // get feedback by item-id func (s *RestServer) getFeedbackByItem(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } itemId := request.PathParameter("item-id") - feedback, err := s.DataClient.GetItemFeedback(itemId) + feedback, err := s.DataClient.GetItemFeedback(ctx, itemId) if err != nil { InternalServerError(response, err) return @@ -628,47 +643,48 @@ func (s *RestServer) getCollaborative(request *restful.Request, response *restfu // 1. If there are recommendations in cache, return cached recommendations. // 2. If there are historical interactions of the users, return similar items. // 3. Otherwise, return fallback recommendation (popular/latest). -func (s *RestServer) Recommend(response *restful.Response, userId, category string, n int, recommenders ...Recommender) ([]string, error) { +func (s *RestServer) Recommend(ctx context.Context, response *restful.Response, userId, category string, n int, recommenders ...Recommender) ([]string, error) { initStart := time.Now() // create context - ctx, err := s.createRecommendContext(userId, category, n) + recommendCtx, err := s.createRecommendContext(ctx, userId, category, n) if err != nil { return nil, errors.Trace(err) } // execute recommenders for _, recommender := range recommenders { - err = recommender(ctx) + err = recommender(recommendCtx) if err != nil { return nil, errors.Trace(err) } } // return recommendations - if len(ctx.results) > n { - ctx.results = ctx.results[:n] + if len(recommendCtx.results) > n { + recommendCtx.results = recommendCtx.results[:n] } totalTime := time.Since(initStart) log.ResponseLogger(response).Info("complete recommendation", - zap.Int("num_from_final", ctx.numFromOffline), - zap.Int("num_from_collaborative", ctx.numFromCollaborative), - zap.Int("num_from_item_based", ctx.numFromItemBased), - zap.Int("num_from_user_based", ctx.numFromUserBased), - zap.Int("num_from_latest", ctx.numFromLatest), - zap.Int("num_from_poplar", ctx.numFromPopular), + zap.Int("num_from_final", recommendCtx.numFromOffline), + zap.Int("num_from_collaborative", recommendCtx.numFromCollaborative), + zap.Int("num_from_item_based", recommendCtx.numFromItemBased), + zap.Int("num_from_user_based", recommendCtx.numFromUserBased), + zap.Int("num_from_latest", recommendCtx.numFromLatest), + zap.Int("num_from_poplar", recommendCtx.numFromPopular), zap.Duration("total_time", totalTime), - zap.Duration("load_final_recommend_time", ctx.loadOfflineRecTime), - zap.Duration("load_col_recommend_time", ctx.loadColRecTime), - zap.Duration("load_hist_time", ctx.loadLoadHistTime), - zap.Duration("item_based_recommend_time", ctx.itemBasedTime), - zap.Duration("user_based_recommend_time", ctx.userBasedTime), - zap.Duration("load_latest_time", ctx.loadLatestTime), - zap.Duration("load_popular_time", ctx.loadPopularTime)) - return ctx.results, nil + zap.Duration("load_final_recommend_time", recommendCtx.loadOfflineRecTime), + zap.Duration("load_col_recommend_time", recommendCtx.loadColRecTime), + zap.Duration("load_hist_time", recommendCtx.loadLoadHistTime), + zap.Duration("item_based_recommend_time", recommendCtx.itemBasedTime), + zap.Duration("user_based_recommend_time", recommendCtx.userBasedTime), + zap.Duration("load_latest_time", recommendCtx.loadLatestTime), + zap.Duration("load_popular_time", recommendCtx.loadPopularTime)) + return recommendCtx.results, nil } type recommendContext struct { + context context.Context response *restful.Response userId string category string @@ -694,9 +710,9 @@ type recommendContext struct { loadPopularTime time.Duration } -func (s *RestServer) createRecommendContext(userId, category string, n int) (*recommendContext, error) { +func (s *RestServer) createRecommendContext(ctx context.Context, userId, category string, n int) (*recommendContext, error) { // pull ignored items - ignoreItems, err := s.CacheClient.GetSortedByScore(cache.Key(cache.IgnoreItems, userId), + ignoreItems, err := s.CacheClient.GetSortedByScore(ctx, cache.Key(cache.IgnoreItems, userId), math.Inf(-1), float64(time.Now().Add(s.Config.Server.ClockError).Unix())) if err != nil { return nil, errors.Trace(err) @@ -710,6 +726,7 @@ func (s *RestServer) createRecommendContext(userId, category string, n int) (*re category: category, n: n, excludeSet: excludeSet, + context: ctx, }, nil } @@ -717,7 +734,7 @@ func (s *RestServer) requireUserFeedback(ctx *recommendContext) error { if ctx.userFeedback == nil { start := time.Now() var err error - ctx.userFeedback, err = s.DataClient.GetUserFeedback(ctx.userId, s.Config.Now()) + ctx.userFeedback, err = s.DataClient.GetUserFeedback(ctx.context, ctx.userId, s.Config.Now()) if err != nil { return errors.Trace(err) } @@ -768,7 +785,7 @@ type Recommender func(ctx *recommendContext) error func (s *RestServer) RecommendOffline(ctx *recommendContext) error { if len(ctx.results) < ctx.n { start := time.Now() - recommendation, err := s.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, ctx.userId, ctx.category), 0, s.Config.Recommend.CacheSize) + recommendation, err := s.CacheClient.GetSorted(ctx.context, cache.Key(cache.OfflineRecommend, ctx.userId, ctx.category), 0, s.Config.Recommend.CacheSize) if err != nil { return errors.Trace(err) } @@ -789,7 +806,7 @@ func (s *RestServer) RecommendOffline(ctx *recommendContext) error { func (s *RestServer) RecommendCollaborative(ctx *recommendContext) error { if len(ctx.results) < ctx.n { start := time.Now() - collaborativeRecommendation, err := s.CacheClient.GetSorted(cache.Key(cache.CollaborativeRecommend, ctx.userId, ctx.category), 0, s.Config.Recommend.CacheSize) + collaborativeRecommendation, err := s.CacheClient.GetSorted(ctx.context, cache.Key(cache.CollaborativeRecommend, ctx.userId, ctx.category), 0, s.Config.Recommend.CacheSize) if err != nil { return errors.Trace(err) } @@ -816,13 +833,13 @@ func (s *RestServer) RecommendUserBased(ctx *recommendContext) error { start := time.Now() candidates := make(map[string]float64) // load similar users - similarUsers, err := s.CacheClient.GetSorted(cache.Key(cache.UserNeighbors, ctx.userId), 0, s.Config.Recommend.CacheSize) + similarUsers, err := s.CacheClient.GetSorted(ctx.context, cache.Key(cache.UserNeighbors, ctx.userId), 0, s.Config.Recommend.CacheSize) if err != nil { return errors.Trace(err) } for _, user := range similarUsers { // load historical feedback - feedbacks, err := s.DataClient.GetUserFeedback(user.Id, s.Config.Now(), s.Config.Recommend.DataSource.PositiveFeedbackTypes...) + feedbacks, err := s.DataClient.GetUserFeedback(ctx.context, user.Id, s.Config.Now(), s.Config.Recommend.DataSource.PositiveFeedbackTypes...) if err != nil { return errors.Trace(err) } @@ -830,7 +847,7 @@ func (s *RestServer) RecommendUserBased(ctx *recommendContext) error { // add unseen items for _, feedback := range feedbacks { if !ctx.excludeSet.Has(feedback.ItemId) { - item, err := s.DataClient.GetItem(feedback.ItemId) + item, err := s.DataClient.GetItem(ctx.context, feedback.ItemId) if err != nil { return errors.Trace(err) } @@ -878,7 +895,7 @@ func (s *RestServer) RecommendItemBased(ctx *recommendContext) error { candidates := make(map[string]float64) for _, feedback := range userFeedback { // load similar items - similarItems, err := s.CacheClient.GetSorted(cache.Key(cache.ItemNeighbors, feedback.ItemId, ctx.category), 0, s.Config.Recommend.CacheSize) + similarItems, err := s.CacheClient.GetSorted(ctx.context, cache.Key(cache.ItemNeighbors, feedback.ItemId, ctx.category), 0, s.Config.Recommend.CacheSize) if err != nil { return errors.Trace(err) } @@ -913,7 +930,7 @@ func (s *RestServer) RecommendLatest(ctx *recommendContext) error { return errors.Trace(err) } start := time.Now() - items, err := s.CacheClient.GetSorted(cache.Key(cache.LatestItems, ctx.category), 0, s.Config.Recommend.CacheSize) + items, err := s.CacheClient.GetSorted(ctx.context, cache.Key(cache.LatestItems, ctx.category), 0, s.Config.Recommend.CacheSize) if err != nil { return errors.Trace(err) } @@ -938,7 +955,7 @@ func (s *RestServer) RecommendPopular(ctx *recommendContext) error { return errors.Trace(err) } start := time.Now() - items, err := s.CacheClient.GetSorted(cache.Key(cache.PopularItems, ctx.category), 0, s.Config.Recommend.CacheSize) + items, err := s.CacheClient.GetSorted(ctx.context, cache.Key(cache.PopularItems, ctx.category), 0, s.Config.Recommend.CacheSize) if err != nil { return errors.Trace(err) } @@ -957,6 +974,10 @@ func (s *RestServer) RecommendPopular(ctx *recommendContext) error { } func (s *RestServer) getRecommend(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } // parse arguments userId := request.PathParameter("user-id") n, err := ParseInt(request, "n", s.Config.Server.DefaultN) @@ -995,7 +1016,7 @@ func (s *RestServer) getRecommend(request *restful.Request, response *restful.Re return } } - results, err := s.Recommend(response, userId, category, offset+n, recommenders...) + results, err := s.Recommend(ctx, response, userId, category, offset+n, recommenders...) if err != nil { InternalServerError(response, err) return @@ -1014,13 +1035,13 @@ func (s *RestServer) getRecommend(request *restful.Request, response *restful.Re }, Timestamp: startTime.Add(writeBackDelay), } - err = s.DataClient.BatchInsertFeedback([]data.Feedback{feedback}, false, false, false) + err = s.DataClient.BatchInsertFeedback(ctx, []data.Feedback{feedback}, false, false, false) if err != nil { InternalServerError(response, err) return } // insert to cache store - err = s.InsertFeedbackToCache([]data.Feedback{feedback}) + err = s.InsertFeedbackToCache(ctx, []data.Feedback{feedback}) if err != nil { InternalServerError(response, err) return @@ -1032,6 +1053,10 @@ func (s *RestServer) getRecommend(request *restful.Request, response *restful.Re } func (s *RestServer) sessionRecommend(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } // parse arguments var feedbacks []Feedback if err := request.ReadEntity(&feedbacks); err != nil { @@ -1076,7 +1101,7 @@ func (s *RestServer) sessionRecommend(request *restful.Request, response *restfu usedFeedbackCount := 0 for _, feedback := range userFeedback { // load similar items - similarItems, err := s.CacheClient.GetSorted(cache.Key(cache.ItemNeighbors, feedback.ItemId, category), 0, s.Config.Recommend.CacheSize) + similarItems, err := s.CacheClient.GetSorted(ctx, cache.Key(cache.ItemNeighbors, feedback.ItemId, category), 0, s.Config.Recommend.CacheSize) if err != nil { BadRequest(response, err) return @@ -1118,18 +1143,22 @@ type Success struct { } func (s *RestServer) insertUser(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } temp := data.User{} // get userInfo from request and put into temp if err := request.ReadEntity(&temp); err != nil { BadRequest(response, err) return } - if err := s.DataClient.BatchInsertUsers([]data.User{temp}); err != nil { + if err := s.DataClient.BatchInsertUsers(ctx, []data.User{temp}); err != nil { InternalServerError(response, err) return } // insert modify timestamp - if err := s.CacheClient.Set(cache.Time(cache.Key(cache.LastModifyUserTime, temp.UserId), time.Now())); err != nil { + if err := s.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastModifyUserTime, temp.UserId), time.Now())); err != nil { InternalServerError(response, err) return } @@ -1137,6 +1166,10 @@ func (s *RestServer) insertUser(request *restful.Request, response *restful.Resp } func (s *RestServer) modifyUser(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } // get user id userId := request.PathParameter("user-id") // modify user @@ -1145,22 +1178,26 @@ func (s *RestServer) modifyUser(request *restful.Request, response *restful.Resp BadRequest(response, err) return } - if err := s.DataClient.ModifyUser(userId, patch); err != nil { + if err := s.DataClient.ModifyUser(ctx, userId, patch); err != nil { InternalServerError(response, err) return } // insert modify timestamp - if err := s.CacheClient.Set(cache.Time(cache.Key(cache.LastModifyUserTime, userId), time.Now())); err != nil { + if err := s.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastModifyUserTime, userId), time.Now())); err != nil { return } Ok(response, Success{RowAffected: 1}) } func (s *RestServer) getUser(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } // get user id userId := request.PathParameter("user-id") // get user - user, err := s.DataClient.GetUser(userId) + user, err := s.DataClient.GetUser(ctx, userId) if err != nil { if errors.Is(err, errors.NotFound) { PageNotFound(response, err) @@ -1173,6 +1210,10 @@ func (s *RestServer) getUser(request *restful.Request, response *restful.Respons } func (s *RestServer) insertUsers(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } var temp []data.User // get param from request and put into temp if err := request.ReadEntity(&temp); err != nil { @@ -1180,7 +1221,7 @@ func (s *RestServer) insertUsers(request *restful.Request, response *restful.Res return } // range temp and achieve user - if err := s.DataClient.BatchInsertUsers(temp); err != nil { + if err := s.DataClient.BatchInsertUsers(ctx, temp); err != nil { InternalServerError(response, err) return } @@ -1189,7 +1230,7 @@ func (s *RestServer) insertUsers(request *restful.Request, response *restful.Res for i, user := range temp { values[i] = cache.Time(cache.Key(cache.LastModifyUserTime, user.UserId), time.Now()) } - if err := s.CacheClient.Set(values...); err != nil { + if err := s.CacheClient.Set(ctx, values...); err != nil { InternalServerError(response, err) return } @@ -1202,6 +1243,10 @@ type UserIterator struct { } func (s *RestServer) getUsers(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } cursor := request.QueryParameter("cursor") n, err := ParseInt(request, "n", s.Config.Server.DefaultN) if err != nil { @@ -1209,7 +1254,7 @@ func (s *RestServer) getUsers(request *restful.Request, response *restful.Respon return } // get all users - cursor, users, err := s.DataClient.GetUsers(cursor, n) + cursor, users, err := s.DataClient.GetUsers(ctx, cursor, n) if err != nil { InternalServerError(response, err) return @@ -1219,9 +1264,13 @@ func (s *RestServer) getUsers(request *restful.Request, response *restful.Respon // delete a user by user-id func (s *RestServer) deleteUser(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } // get user-id and put into temp userId := request.PathParameter("user-id") - if err := s.DataClient.DeleteUser(userId); err != nil { + if err := s.DataClient.DeleteUser(ctx, userId); err != nil { InternalServerError(response, err) return } @@ -1230,9 +1279,13 @@ func (s *RestServer) deleteUser(request *restful.Request, response *restful.Resp // get feedback by user-id with feedback type func (s *RestServer) getTypedFeedbackByUser(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } feedbackType := request.PathParameter("feedback-type") userId := request.PathParameter("user-id") - feedback, err := s.DataClient.GetUserFeedback(userId, s.Config.Now(), feedbackType) + feedback, err := s.DataClient.GetUserFeedback(ctx, userId, s.Config.Now(), feedbackType) if err != nil { InternalServerError(response, err) return @@ -1242,8 +1295,12 @@ func (s *RestServer) getTypedFeedbackByUser(request *restful.Request, response * // get feedback by user-id func (s *RestServer) getFeedbackByUser(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } userId := request.PathParameter("user-id") - feedback, err := s.DataClient.GetUserFeedback(userId, s.Config.Now()) + feedback, err := s.DataClient.GetUserFeedback(ctx, userId, s.Config.Now()) if err != nil { InternalServerError(response, err) return @@ -1261,7 +1318,7 @@ type Item struct { Comment string } -func (s *RestServer) batchInsertItems(response *restful.Response, temp []Item) { +func (s *RestServer) batchInsertItems(ctx context.Context, response *restful.Response, temp []Item) { var ( count int items = make([]data.Item, 0, len(temp)) @@ -1277,7 +1334,7 @@ func (s *RestServer) batchInsertItems(response *restful.Response, temp []Item) { ) // load existed items start := time.Now() - existedItems, err := s.DataClient.BatchGetItems(lo.Map(temp, func(t Item, i int) string { + existedItems, err := s.DataClient.BatchGetItems(ctx, lo.Map(temp, func(t Item, i int) string { return t.ItemId })) if err != nil { @@ -1327,7 +1384,7 @@ func (s *RestServer) batchInsertItems(response *restful.Response, temp []Item) { // insert items start = time.Now() - if err = s.DataClient.BatchInsertItems(items); err != nil { + if err = s.DataClient.BatchInsertItems(ctx, items); err != nil { InternalServerError(response, err) return } @@ -1341,12 +1398,12 @@ func (s *RestServer) batchInsertItems(response *restful.Response, temp []Item) { values[i] = cache.Time(cache.Key(cache.LastModifyItemTime, item.ItemId), time.Now()) categories.Add(item.Categories...) } - if err = s.CacheClient.Set(values...); err != nil { + if err = s.CacheClient.Set(ctx, values...); err != nil { InternalServerError(response, err) return } // insert categories - if err = s.CacheClient.AddSet(cache.ItemCategories, categories.List()...); err != nil { + if err = s.CacheClient.AddSet(ctx, cache.ItemCategories, categories.List()...); err != nil { InternalServerError(response, err) return } @@ -1365,26 +1422,38 @@ func (s *RestServer) batchInsertItems(response *restful.Response, temp []Item) { } func (s *RestServer) insertItems(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } var items []Item if err := request.ReadEntity(&items); err != nil { BadRequest(response, err) return } // Insert items - s.batchInsertItems(response, items) + s.batchInsertItems(ctx, response, items) } func (s *RestServer) insertItem(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } var item Item var err error if err = request.ReadEntity(&item); err != nil { BadRequest(response, err) return } - s.batchInsertItems(response, []Item{item}) + s.batchInsertItems(ctx, response, []Item{item}) } func (s *RestServer) modifyItem(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } itemId := request.PathParameter("item-id") var patch data.ItemPatch if err := request.ReadEntity(&patch); err != nil { @@ -1402,7 +1471,7 @@ func (s *RestServer) modifyItem(request *restful.Request, response *restful.Resp } // insert new timestamp to the latest scores if patch.Timestamp != nil || patch.Categories != nil { - item, err := s.DataClient.GetItem(itemId) + item, err := s.DataClient.GetItem(ctx, itemId) if err != nil { InternalServerError(response, err) return @@ -1414,12 +1483,12 @@ func (s *RestServer) modifyItem(request *restful.Request, response *restful.Resp popularScore) } // modify item - if err := s.DataClient.ModifyItem(itemId, patch); err != nil { + if err := s.DataClient.ModifyItem(ctx, itemId, patch); err != nil { InternalServerError(response, err) return } // insert modify timestamp - if err := s.CacheClient.Set(cache.Time(cache.Key(cache.LastModifyItemTime, itemId), time.Now())); err != nil { + if err := s.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastModifyItemTime, itemId), time.Now())); err != nil { return } // refresh cache @@ -1437,13 +1506,17 @@ type ItemIterator struct { } func (s *RestServer) getItems(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } cursor := request.QueryParameter("cursor") n, err := ParseInt(request, "n", s.Config.Server.DefaultN) if err != nil { BadRequest(response, err) return } - cursor, items, err := s.DataClient.GetItems(cursor, n, nil) + cursor, items, err := s.DataClient.GetItems(ctx, cursor, n, nil) if err != nil { InternalServerError(response, err) return @@ -1452,10 +1525,14 @@ func (s *RestServer) getItems(request *restful.Request, response *restful.Respon } func (s *RestServer) getItem(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } // Get item id itemId := request.PathParameter("item-id") // Get item - item, err := s.DataClient.GetItem(itemId) + item, err := s.DataClient.GetItem(ctx, itemId) if err != nil { if errors.Is(err, errors.NotFound) { PageNotFound(response, err) @@ -1468,9 +1545,13 @@ func (s *RestServer) getItem(request *restful.Request, response *restful.Respons } func (s *RestServer) deleteItem(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } itemId := request.PathParameter("item-id") // delete item - if err := s.DataClient.DeleteItem(itemId); err != nil { + if err := s.DataClient.DeleteItem(ctx, itemId); err != nil { InternalServerError(response, err) return } @@ -1483,11 +1564,15 @@ func (s *RestServer) deleteItem(request *restful.Request, response *restful.Resp } func (s *RestServer) insertItemCategory(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } // Get item id and category itemId := request.PathParameter("item-id") category := request.PathParameter("category") // Insert category - item, err := s.DataClient.GetItem(itemId) + item, err := s.DataClient.GetItem(ctx, itemId) if err != nil { InternalServerError(response, err) return @@ -1495,7 +1580,7 @@ func (s *RestServer) insertItemCategory(request *restful.Request, response *rest if !funk.ContainsString(item.Categories, category) { item.Categories = append(item.Categories, category) } - err = s.DataClient.BatchInsertItems([]data.Item{item}) + err = s.DataClient.BatchInsertItems(ctx, []data.Item{item}) if err != nil { InternalServerError(response, err) return @@ -1512,11 +1597,15 @@ func (s *RestServer) insertItemCategory(request *restful.Request, response *rest } func (s *RestServer) deleteItemCategory(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } // Get item id and category itemId := request.PathParameter("item-id") category := request.PathParameter("category") // Delete category - item, err := s.DataClient.GetItem(itemId) + item, err := s.DataClient.GetItem(ctx, itemId) if err != nil { InternalServerError(response, err) return @@ -1528,7 +1617,7 @@ func (s *RestServer) deleteItemCategory(request *restful.Request, response *rest } } item.Categories = categories - err = s.DataClient.BatchInsertItems([]data.Item{item}) + err = s.DataClient.BatchInsertItems(ctx, []data.Item{item}) if err != nil { InternalServerError(response, err) return @@ -1564,6 +1653,10 @@ func (f Feedback) ToDataFeedback() (data.Feedback, error) { func (s *RestServer) insertFeedback(overwrite bool) func(request *restful.Request, response *restful.Response) { return func(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } // add ratings var feedbackLiterTime []Feedback if err := request.ReadEntity(&feedbackLiterTime); err != nil { @@ -1585,7 +1678,7 @@ func (s *RestServer) insertFeedback(overwrite bool) func(request *restful.Reques } } // insert feedback to data store - err = s.DataClient.BatchInsertFeedback(feedback, + err = s.DataClient.BatchInsertFeedback(ctx, feedback, s.Config.Server.AutoInsertUser, s.Config.Server.AutoInsertItem, overwrite) if err != nil { @@ -1593,7 +1686,7 @@ func (s *RestServer) insertFeedback(overwrite bool) func(request *restful.Reques return } // insert feedback to cache store - if err = s.InsertFeedbackToCache(feedback); err != nil { + if err = s.InsertFeedbackToCache(ctx, feedback); err != nil { InternalServerError(response, err) return } @@ -1604,7 +1697,7 @@ func (s *RestServer) insertFeedback(overwrite bool) func(request *restful.Reques for _, itemId := range items.List() { values = append(values, cache.Time(cache.Key(cache.LastModifyItemTime, itemId), time.Now())) } - if err = s.CacheClient.Set(values...); err != nil { + if err = s.CacheClient.Set(ctx, values...); err != nil { InternalServerError(response, err) return } @@ -1620,6 +1713,10 @@ type FeedbackIterator struct { } func (s *RestServer) getFeedback(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } // Parse parameters cursor := request.QueryParameter("cursor") n, err := ParseInt(request, "n", s.Config.Server.DefaultN) @@ -1627,7 +1724,7 @@ func (s *RestServer) getFeedback(request *restful.Request, response *restful.Res BadRequest(response, err) return } - cursor, feedback, err := s.DataClient.GetFeedback(cursor, n, nil, s.Config.Now()) + cursor, feedback, err := s.DataClient.GetFeedback(ctx, cursor, n, nil, s.Config.Now()) if err != nil { InternalServerError(response, err) return @@ -1636,6 +1733,10 @@ func (s *RestServer) getFeedback(request *restful.Request, response *restful.Res } func (s *RestServer) getTypedFeedback(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } // Parse parameters feedbackType := request.PathParameter("feedback-type") cursor := request.QueryParameter("cursor") @@ -1644,7 +1745,7 @@ func (s *RestServer) getTypedFeedback(request *restful.Request, response *restfu BadRequest(response, err) return } - cursor, feedback, err := s.DataClient.GetFeedback(cursor, n, nil, s.Config.Now(), feedbackType) + cursor, feedback, err := s.DataClient.GetFeedback(ctx, cursor, n, nil, s.Config.Now(), feedbackType) if err != nil { InternalServerError(response, err) return @@ -1653,10 +1754,14 @@ func (s *RestServer) getTypedFeedback(request *restful.Request, response *restfu } func (s *RestServer) getUserItemFeedback(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } // Parse parameters userId := request.PathParameter("user-id") itemId := request.PathParameter("item-id") - if feedback, err := s.DataClient.GetUserItemFeedback(userId, itemId); err != nil { + if feedback, err := s.DataClient.GetUserItemFeedback(ctx, userId, itemId); err != nil { InternalServerError(response, err) } else { Ok(response, feedback) @@ -1664,10 +1769,14 @@ func (s *RestServer) getUserItemFeedback(request *restful.Request, response *res } func (s *RestServer) deleteUserItemFeedback(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } // Parse parameters userId := request.PathParameter("user-id") itemId := request.PathParameter("item-id") - if deleteCount, err := s.DataClient.DeleteUserItemFeedback(userId, itemId); err != nil { + if deleteCount, err := s.DataClient.DeleteUserItemFeedback(ctx, userId, itemId); err != nil { InternalServerError(response, err) } else { Ok(response, Success{RowAffected: deleteCount}) @@ -1675,11 +1784,15 @@ func (s *RestServer) deleteUserItemFeedback(request *restful.Request, response * } func (s *RestServer) getTypedUserItemFeedback(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } // Parse parameters feedbackType := request.PathParameter("feedback-type") userId := request.PathParameter("user-id") itemId := request.PathParameter("item-id") - if feedback, err := s.DataClient.GetUserItemFeedback(userId, itemId, feedbackType); err != nil { + if feedback, err := s.DataClient.GetUserItemFeedback(ctx, userId, itemId, feedbackType); err != nil { InternalServerError(response, err) } else if feedbackType == "" { Text(response, "{}") @@ -1689,11 +1802,15 @@ func (s *RestServer) getTypedUserItemFeedback(request *restful.Request, response } func (s *RestServer) deleteTypedUserItemFeedback(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } // Parse parameters feedbackType := request.PathParameter("feedback-type") userId := request.PathParameter("user-id") itemId := request.PathParameter("item-id") - if deleteCount, err := s.DataClient.DeleteUserItemFeedback(userId, itemId, feedbackType); err != nil { + if deleteCount, err := s.DataClient.DeleteUserItemFeedback(ctx, userId, itemId, feedbackType); err != nil { InternalServerError(response, err) } else { Ok(response, Success{deleteCount}) @@ -1733,8 +1850,8 @@ func (m Measurement) GetScore() cache.Scored { return cache.Scored{Id: string(buf), Score: float64(m.Timestamp.Unix())} } -func (s *RestServer) GetMeasurements(name string, n int) ([]Measurement, error) { - scores, err := s.CacheClient.GetSorted(cache.Key(cache.Measurements, name), 0, n-1) +func (s *RestServer) GetMeasurements(ctx context.Context, name string, n int) ([]Measurement, error) { + scores, err := s.CacheClient.GetSorted(ctx, cache.Key(cache.Measurements, name), 0, n-1) if err != nil { return nil, errors.Trace(err) } @@ -1749,20 +1866,24 @@ func (s *RestServer) GetMeasurements(name string, n int) ([]Measurement, error) return measurements, nil } -func (s *RestServer) InsertMeasurement(measurements ...Measurement) error { +func (s *RestServer) InsertMeasurement(ctx context.Context, measurements ...Measurement) error { sortedSets := lo.Map(measurements, func(m Measurement, _ int) cache.SortedSet { name := cache.Key(cache.Measurements, m.Name) score := m.GetScore() - err := s.CacheClient.RemSortedByScore(name, score.Score, score.Score) + err := s.CacheClient.RemSortedByScore(ctx, name, score.Score, score.Score) if err != nil { log.Logger().Warn("failed to remove stale measurement", zap.Error(err)) } return cache.Sorted(name, []cache.Scored{score}) }) - return s.CacheClient.AddSorted(sortedSets...) + return s.CacheClient.AddSorted(ctx, sortedSets...) } func (s *RestServer) getMeasurements(request *restful.Request, response *restful.Response) { + ctx := context.Background() + if request != nil && request.Request != nil { + ctx = request.Request.Context() + } // Parse parameters name := request.PathParameter("name") n, err := ParseInt(request, "n", 100) @@ -1770,7 +1891,7 @@ func (s *RestServer) getMeasurements(request *restful.Request, response *restful BadRequest(response, err) return } - measurements, err := s.GetMeasurements(name, n) + measurements, err := s.GetMeasurements(ctx, name, n) if err != nil { InternalServerError(response, err) return @@ -1821,13 +1942,13 @@ func Text(response *restful.Response, content string) { } // InsertFeedbackToCache inserts feedback to cache. -func (s *RestServer) InsertFeedbackToCache(feedback []data.Feedback) error { +func (s *RestServer) InsertFeedbackToCache(ctx context.Context, feedback []data.Feedback) error { if !s.Config.Recommend.Replacement.EnableReplacement { sortedSets := make([]cache.SortedSet, len(feedback)) for i, v := range feedback { sortedSets[i] = cache.Sorted(cache.Key(cache.IgnoreItems, v.UserId), []cache.Scored{{Id: v.ItemId, Score: float64(v.Timestamp.Unix())}}) } - if err := s.CacheClient.AddSorted(sortedSets...); err != nil { + if err := s.CacheClient.AddSorted(ctx, sortedSets...); err != nil { return errors.Trace(err) } } diff --git a/server/rest_test.go b/server/rest_test.go index 102325704..aa68afaaf 100644 --- a/server/rest_test.go +++ b/server/rest_test.go @@ -14,6 +14,7 @@ package server import ( + "context" "encoding/json" "github.com/alicebob/miniredis/v2" "github.com/emicklei/go-restful/v3" @@ -172,6 +173,7 @@ func TestServer_Users(t *testing.T) { } func TestServer_Items(t *testing.T) { + ctx := context.Background() s := newMockServer(t) defer s.Close(t) // Items @@ -213,7 +215,7 @@ func TestServer_Items(t *testing.T) { }, } // insert popular scores - err := s.CacheClient.SetSorted(cache.PopularItems, []cache.Scored{ + err := s.CacheClient.SetSorted(ctx, cache.PopularItems, []cache.Scored{ {Id: "0", Score: 10}, {Id: "2", Score: 12}, {Id: "4", Score: 14}, @@ -316,7 +318,7 @@ func TestServer_Items(t *testing.T) { })). End() // get categories - categories, err := s.CacheClient.GetSet(cache.ItemCategories) + categories, err := s.CacheClient.GetSet(ctx, cache.ItemCategories) assert.NoError(t, err) assert.Equal(t, []string{"*"}, categories) @@ -607,6 +609,7 @@ func TestServer_Items(t *testing.T) { } func TestServer_Feedback(t *testing.T) { + ctx := context.Background() s := newMockServer(t) defer s.Close(t) // Insert ret @@ -724,7 +727,7 @@ func TestServer_Feedback(t *testing.T) { Status(http.StatusOK). Body(`{"RowAffected": 1}`). End() - ret, err := s.DataClient.GetUserFeedback("0", s.Config.Now(), "click") + ret, err := s.DataClient.GetUserFeedback(ctx, "0", s.Config.Now(), "click") assert.NoError(t, err) assert.Equal(t, 1, len(ret)) assert.Equal(t, "override", ret[0].Comment) @@ -741,7 +744,7 @@ func TestServer_Feedback(t *testing.T) { Status(http.StatusOK). Body(`{"RowAffected": 1}`). End() - ret, err = s.DataClient.GetUserFeedback("0", s.Config.Now(), "click") + ret, err = s.DataClient.GetUserFeedback(ctx, "0", s.Config.Now(), "click") assert.NoError(t, err) assert.Equal(t, 1, len(ret)) assert.Equal(t, "override", ret[0].Comment) @@ -759,6 +762,7 @@ func TestServer_Feedback(t *testing.T) { } func TestServer_Sort(t *testing.T) { + ctx := context.Background() s := newMockServer(t) defer s.Close(t) type ListOperator struct { @@ -789,7 +793,7 @@ func TestServer_Sort(t *testing.T) { {strconv.Itoa(i) + "3", 97}, {strconv.Itoa(i) + "4", 96}, } - err := s.CacheClient.SetSorted(operator.Key, scores) + err := s.CacheClient.SetSorted(ctx, operator.Key, scores) assert.NoError(t, err) err = NewCacheModification(s.CacheClient, s.HiddenItemsManager).HideItem(strconv.Itoa(i) + "3").Exec() assert.NoError(t, err) @@ -897,6 +901,7 @@ func TestServer_DeleteFeedback(t *testing.T) { } func TestServer_Measurement(t *testing.T) { + ctx := context.Background() s := newMockServer(t) defer s.Close(t) measurements := []Measurement{ @@ -907,7 +912,7 @@ func TestServer_Measurement(t *testing.T) { {"Test_NDCG", time.Date(2004, 1, 1, 1, 1, 1, 0, time.UTC).Local(), 4}, {"Test_Recall", time.Date(2000, 1, 1, 1, 1, 1, 0, time.UTC).Local(), 1}, } - err := s.InsertMeasurement(measurements...) + err := s.InsertMeasurement(ctx, measurements...) assert.NoError(t, err) apitest.New(). Handler(s.handler). @@ -933,15 +938,16 @@ func TestServer_Measurement(t *testing.T) { } func TestServer_GetRecommends(t *testing.T) { + ctx := context.Background() s := newMockServer(t) defer s.Close(t) // insert hidden items - err := s.CacheClient.SetSorted(cache.Key(cache.OfflineRecommend, "0"), []cache.Scored{{"0", 100}}) + err := s.CacheClient.SetSorted(ctx, cache.Key(cache.OfflineRecommend, "0"), []cache.Scored{{"0", 100}}) assert.NoError(t, err) err = NewCacheModification(s.CacheClient, s.HiddenItemsManager).HideItem("0").Exec() assert.NoError(t, err) // insert recommendation - err = s.CacheClient.SetSorted(cache.Key(cache.OfflineRecommend, "0"), []cache.Scored{ + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.OfflineRecommend, "0"), []cache.Scored{ {Id: "1", Score: 99}, {Id: "2", Score: 98}, {Id: "3", Score: 97}, @@ -1041,16 +1047,17 @@ func TestServer_GetRecommends(t *testing.T) { } func TestServer_GetRecommends_Replacement(t *testing.T) { + ctx := context.Background() s := newMockServer(t) s.Config.Recommend.Replacement.EnableReplacement = true defer s.Close(t) // insert hidden items - err := s.CacheClient.SetSorted(cache.Key(cache.OfflineRecommend, "0"), []cache.Scored{{"0", 100}}) + err := s.CacheClient.SetSorted(ctx, cache.Key(cache.OfflineRecommend, "0"), []cache.Scored{{"0", 100}}) assert.NoError(t, err) err = NewCacheModification(s.CacheClient, s.HiddenItemsManager).HideItem("0").Exec() assert.NoError(t, err) // insert recommendation - err = s.CacheClient.SetSorted(cache.Key(cache.OfflineRecommend, "0"), []cache.Scored{ + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.OfflineRecommend, "0"), []cache.Scored{ {Id: "1", Score: 99}, {Id: "2", Score: 98}, {Id: "3", Score: 97}, @@ -1090,12 +1097,13 @@ func TestServer_GetRecommends_Replacement(t *testing.T) { } func TestServer_GetRecommends_Fallback_ItemBasedSimilar(t *testing.T) { + ctx := context.Background() s := newMockServer(t) s.Config.Recommend.Online.NumFeedbackFallbackItemBased = 4 s.Config.Recommend.DataSource.PositiveFeedbackTypes = []string{"a"} defer s.Close(t) // insert recommendation - err := s.CacheClient.SetSorted(cache.Key(cache.OfflineRecommend, "0"), + err := s.CacheClient.SetSorted(ctx, cache.Key(cache.OfflineRecommend, "0"), []cache.Scored{{"1", 99}, {"2", 98}, {"3", 97}, {"4", 96}}) assert.NoError(t, err) // insert feedback @@ -1117,25 +1125,25 @@ func TestServer_GetRecommends_Fallback_ItemBasedSimilar(t *testing.T) { End() // insert similar items - err = s.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "1"), []cache.Scored{ + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "1"), []cache.Scored{ {"2", 100000}, {"9", 1}, }) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "2"), []cache.Scored{ + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "2"), []cache.Scored{ {"3", 100000}, {"8", 1}, {"9", 1}, }) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "3"), []cache.Scored{ + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "3"), []cache.Scored{ {"4", 100000}, {"7", 1}, {"8", 1}, {"9", 1}, }) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "4"), []cache.Scored{ + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "4"), []cache.Scored{ {"1", 100000}, {"6", 1}, {"7", 1}, @@ -1143,7 +1151,7 @@ func TestServer_GetRecommends_Fallback_ItemBasedSimilar(t *testing.T) { {"9", 1}, }) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "5"), []cache.Scored{ + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "5"), []cache.Scored{ {"1", 1}, {"6", 1}, {"7", 100000}, @@ -1153,21 +1161,21 @@ func TestServer_GetRecommends_Fallback_ItemBasedSimilar(t *testing.T) { assert.NoError(t, err) // insert similar items of category * - err = s.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "1", "*"), []cache.Scored{ + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "1", "*"), []cache.Scored{ {"9", 1}, }) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "2", "*"), []cache.Scored{ + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "2", "*"), []cache.Scored{ {"3", 100000}, {"9", 1}, }) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "3", "*"), []cache.Scored{ + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "3", "*"), []cache.Scored{ {"7", 1}, {"9", 1}, }) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "4", "*"), []cache.Scored{ + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "4", "*"), []cache.Scored{ {"1", 100000}, {"7", 1}, {"9", 1}, @@ -1202,10 +1210,11 @@ func TestServer_GetRecommends_Fallback_ItemBasedSimilar(t *testing.T) { } func TestServer_GetRecommends_Fallback_UserBasedSimilar(t *testing.T) { + ctx := context.Background() s := newMockServer(t) defer s.Close(t) // insert recommendation - err := s.CacheClient.SetSorted(cache.Key(cache.OfflineRecommend, "0"), + err := s.CacheClient.SetSorted(ctx, cache.Key(cache.OfflineRecommend, "0"), []cache.Scored{{"1", 99}, {"2", 98}, {"3", 97}, {"4", 96}}) assert.NoError(t, err) // insert feedback @@ -1225,28 +1234,28 @@ func TestServer_GetRecommends_Fallback_UserBasedSimilar(t *testing.T) { Body(`{"RowAffected": 4}`). End() // insert similar users - err = s.CacheClient.SetSorted(cache.Key(cache.UserNeighbors, "0"), []cache.Scored{ + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.UserNeighbors, "0"), []cache.Scored{ {"1", 2}, {"2", 1.5}, {"3", 1}, }) assert.NoError(t, err) - err = s.DataClient.BatchInsertFeedback([]data.Feedback{ + err = s.DataClient.BatchInsertFeedback(ctx, []data.Feedback{ {FeedbackKey: data.FeedbackKey{FeedbackType: "a", UserId: "1", ItemId: "11"}}, }, true, true, true) assert.NoError(t, err) - err = s.DataClient.BatchInsertFeedback([]data.Feedback{ + err = s.DataClient.BatchInsertFeedback(ctx, []data.Feedback{ {FeedbackKey: data.FeedbackKey{FeedbackType: "a", UserId: "2", ItemId: "12"}}, {FeedbackKey: data.FeedbackKey{FeedbackType: "a", UserId: "2", ItemId: "48"}}, }, true, true, true) assert.NoError(t, err) - err = s.DataClient.BatchInsertFeedback([]data.Feedback{ + err = s.DataClient.BatchInsertFeedback(ctx, []data.Feedback{ {FeedbackKey: data.FeedbackKey{FeedbackType: "a", UserId: "3", ItemId: "13"}}, {FeedbackKey: data.FeedbackKey{FeedbackType: "a", UserId: "3", ItemId: "48"}}, }, true, true, true) assert.NoError(t, err) // insert categorized items - err = s.DataClient.BatchInsertItems([]data.Item{ + err = s.DataClient.BatchInsertItems(ctx, []data.Item{ {ItemId: "12", Categories: []string{"*"}}, {ItemId: "48", Categories: []string{"*"}}, }) @@ -1278,34 +1287,35 @@ func TestServer_GetRecommends_Fallback_UserBasedSimilar(t *testing.T) { } func TestServer_GetRecommends_Fallback_PreCached(t *testing.T) { + ctx := context.Background() s := newMockServer(t) defer s.Close(t) // insert offline recommendation - err := s.CacheClient.SetSorted(cache.Key(cache.OfflineRecommend, "0"), + err := s.CacheClient.SetSorted(ctx, cache.Key(cache.OfflineRecommend, "0"), []cache.Scored{{"1", 99}, {"2", 98}, {"3", 97}, {"4", 96}}) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.Key(cache.OfflineRecommend, "0", "*"), + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.OfflineRecommend, "0", "*"), []cache.Scored{{"101", 99}, {"102", 98}, {"103", 97}, {"104", 96}}) assert.NoError(t, err) // insert latest - err = s.CacheClient.SetSorted(cache.LatestItems, + err = s.CacheClient.SetSorted(ctx, cache.LatestItems, []cache.Scored{{"5", 95}, {"6", 94}, {"7", 93}, {"8", 92}}) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.Key(cache.LatestItems, "*"), + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.LatestItems, "*"), []cache.Scored{{"105", 95}, {"106", 94}, {"107", 93}, {"108", 92}}) assert.NoError(t, err) // insert popular - err = s.CacheClient.SetSorted(cache.PopularItems, + err = s.CacheClient.SetSorted(ctx, cache.PopularItems, []cache.Scored{{"9", 91}, {"10", 90}, {"11", 89}, {"12", 88}}) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.Key(cache.PopularItems, "*"), + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.PopularItems, "*"), []cache.Scored{{"109", 91}, {"110", 90}, {"111", 89}, {"112", 88}}) assert.NoError(t, err) // insert collaborative filtering - err = s.CacheClient.SetSorted(cache.Key(cache.CollaborativeRecommend, "0"), + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.CollaborativeRecommend, "0"), []cache.Scored{{"13", 79}, {"14", 78}, {"15", 77}, {"16", 76}}) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.Key(cache.CollaborativeRecommend, "0", "*"), + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.CollaborativeRecommend, "0", "*"), []cache.Scored{{"113", 79}, {"114", 78}, {"115", 77}, {"116", 76}}) assert.NoError(t, err) // test popular fallback @@ -1395,6 +1405,7 @@ func TestServer_GetRecommends_Fallback_PreCached(t *testing.T) { } func TestServer_SessionRecommend(t *testing.T) { + ctx := context.Background() s := newMockServer(t) s.Config.Recommend.Online.NumFeedbackFallbackItemBased = 4 s.Config.Recommend.DataSource.PositiveFeedbackTypes = []string{"a"} @@ -1412,26 +1423,26 @@ func TestServer_SessionRecommend(t *testing.T) { End() // insert similar items - err := s.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "1"), []cache.Scored{ + err := s.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "1"), []cache.Scored{ {"2", 100000}, {"9", 1}, {"100", 100000}, }) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "2"), []cache.Scored{ + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "2"), []cache.Scored{ {"3", 100000}, {"8", 1}, {"9", 1}, }) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "3"), []cache.Scored{ + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "3"), []cache.Scored{ {"4", 100000}, {"7", 1}, {"8", 1}, {"9", 1}, }) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "4"), []cache.Scored{ + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "4"), []cache.Scored{ {"1", 100000}, {"6", 1}, {"7", 1}, @@ -1439,7 +1450,7 @@ func TestServer_SessionRecommend(t *testing.T) { {"9", 1}, }) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "5"), []cache.Scored{ + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "5"), []cache.Scored{ {"1", 1}, {"6", 1}, {"7", 100000}, @@ -1449,21 +1460,21 @@ func TestServer_SessionRecommend(t *testing.T) { assert.NoError(t, err) // insert similar items of category * - err = s.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "1", "*"), []cache.Scored{ + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "1", "*"), []cache.Scored{ {"9", 1}, }) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "2", "*"), []cache.Scored{ + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "2", "*"), []cache.Scored{ {"3", 100000}, {"9", 1}, }) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "3", "*"), []cache.Scored{ + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "3", "*"), []cache.Scored{ {"7", 1}, {"9", 1}, }) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "4", "*"), []cache.Scored{ + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "4", "*"), []cache.Scored{ {"1", 100000}, {"7", 1}, {"9", 1}, @@ -1518,6 +1529,7 @@ func TestServer_SessionRecommend(t *testing.T) { } func TestServer_Visibility(t *testing.T) { + ctx := context.Background() s := newMockServer(t) defer s.Close(t) @@ -1548,13 +1560,13 @@ func TestServer_Visibility(t *testing.T) { scores = append(scores, cache.Scored{Id: strconv.Itoa(i), Score: float64(time.Date(1989, 6, i+1, 1, 1, 1, 1, time.UTC).Unix())}) } lo.Reverse(scores) - err := s.CacheClient.SetSorted(cache.LatestItems, scores) + err := s.CacheClient.SetSorted(ctx, cache.LatestItems, scores) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.PopularItems, scores) + err = s.CacheClient.SetSorted(ctx, cache.PopularItems, scores) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "100"), scores) + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "100"), scores) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.Key(cache.OfflineRecommend, "100"), scores) + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.OfflineRecommend, "100"), scores) assert.NoError(t, err) // delete item @@ -1690,13 +1702,13 @@ func TestServer_Visibility(t *testing.T) { End() // insert cache - err = s.CacheClient.SetSorted(cache.Key(cache.LatestItems, "a"), scores) + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.LatestItems, "a"), scores) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.Key(cache.PopularItems, "a"), scores) + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.PopularItems, "a"), scores) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "100", "a"), scores) + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "100", "a"), scores) assert.NoError(t, err) - err = s.CacheClient.SetSorted(cache.Key(cache.OfflineRecommend, "100", "a"), scores) + err = s.CacheClient.SetSorted(ctx, cache.Key(cache.OfflineRecommend, "100", "a"), scores) assert.NoError(t, err) // delete category diff --git a/server/server.go b/server/server.go index 329284dcb..1ae956c85 100644 --- a/server/server.go +++ b/server/server.go @@ -29,6 +29,8 @@ import ( "github.com/zhenghaoz/gorse/protocol" "github.com/zhenghaoz/gorse/storage/cache" "github.com/zhenghaoz/gorse/storage/data" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -41,6 +43,7 @@ import ( // Server manages states of a server node. type Server struct { RestServer + traceConfig config.TracingConfig cachePath string cachePrefix string dataPath string @@ -167,6 +170,19 @@ func (s *Server) Sync() { s.cachePrefix = s.Config.Database.TablePrefix } + // create trace provider + if !s.traceConfig.Equal(s.Config.Tracing) { + log.Logger().Info("create trace provider", zap.Any("tracing_config", s.Config.Tracing)) + tp, err := s.Config.Tracing.NewTracerProvider() + if err != nil { + log.Logger().Fatal("failed to create trace provider", zap.Error(err)) + } + otel.SetTracerProvider(tp) + otel.SetErrorHandler(log.GetErrorHandler()) + otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})) + s.traceConfig = s.Config.Tracing + } + sleep: if s.testMode { return @@ -207,8 +223,9 @@ func newPopularItemsCacheForTest(s *RestServer) *PopularItemsCache { } func (sc *PopularItemsCache) sync() { + ctx := context.Background() // load popular items - items, err := sc.server.CacheClient.GetSorted(cache.Key(cache.PopularItems), 0, -1) + items, err := sc.server.CacheClient.GetSorted(ctx, cache.Key(cache.PopularItems), 0, -1) if err != nil { if !errors.Is(err, errors.NotAssigned) { log.Logger().Error("failed to get popular items", zap.Error(err)) @@ -267,9 +284,10 @@ func newHiddenItemsManagerForTest(s *RestServer) *HiddenItemsManager { } func (hc *HiddenItemsManager) sync() { + ctx := context.Background() ts := time.Now() // load categories - categories, err := hc.server.CacheClient.GetSet(cache.ItemCategories) + categories, err := hc.server.CacheClient.GetSet(ctx, cache.ItemCategories) if err != nil { if !errors.Is(err, errors.NotAssigned) { log.Logger().Error("failed to load item categories", zap.Error(err)) @@ -277,7 +295,7 @@ func (hc *HiddenItemsManager) sync() { return } // load hidden items - score, err := hc.server.CacheClient.GetSortedByScore(cache.HiddenItemsV2, math.Inf(-1), float64(ts.Unix())) + score, err := hc.server.CacheClient.GetSortedByScore(ctx, cache.HiddenItemsV2, math.Inf(-1), float64(ts.Unix())) if err != nil { if !errors.Is(err, errors.NotAssigned) { log.Logger().Error("failed to load hidden items", zap.Error(err)) @@ -287,7 +305,7 @@ func (hc *HiddenItemsManager) sync() { hiddenItems := strset.New(cache.RemoveScores(score)...) // load hidden items in categories for _, category := range categories { - score, err = hc.server.CacheClient.GetSortedByScore(cache.Key(cache.HiddenItemsV2, category), math.Inf(-1), float64(ts.Unix())) + score, err = hc.server.CacheClient.GetSortedByScore(ctx, cache.Key(cache.HiddenItemsV2, category), math.Inf(-1), float64(ts.Unix())) if err != nil { if !errors.Is(err, errors.NotAssigned) { log.Logger().Error("failed to load categorized hidden items", zap.Error(err)) @@ -303,6 +321,7 @@ func (hc *HiddenItemsManager) sync() { } func (hc *HiddenItemsManager) IsHidden(members []string, category string) ([]bool, error) { + ctx := context.Background() if hc.test { hc.sync() } @@ -319,7 +338,7 @@ func (hc *HiddenItemsManager) IsHidden(members []string, category string) ([]boo } } // load delta hidden items - score, err := hc.server.CacheClient.GetSortedByScore(cache.HiddenItemsV2, float64(updateTime.Unix()), float64(time.Now().Unix())) + score, err := hc.server.CacheClient.GetSortedByScore(ctx, cache.HiddenItemsV2, float64(updateTime.Unix()), float64(time.Now().Unix())) if err != nil { return nil, errors.Trace(err) } @@ -327,7 +346,7 @@ func (hc *HiddenItemsManager) IsHidden(members []string, category string) ([]boo // load delta hidden items in category deltaHiddenItemsInCategory := strset.New() if category != "" { - score, err = hc.server.CacheClient.GetSortedByScore(cache.Key(cache.HiddenItemsV2, category), float64(updateTime.Unix()), float64(time.Now().Unix())) + score, err = hc.server.CacheClient.GetSortedByScore(ctx, cache.Key(cache.HiddenItemsV2, category), float64(updateTime.Unix()), float64(time.Now().Unix())) if err != nil { return nil, errors.Trace(err) } @@ -458,13 +477,14 @@ func (cm *CacheModification) addItem(itemId string, categories []string, latest, } func (cm *CacheModification) Exec() error { + ctx := context.Background() if len(cm.deletion) > 0 { - if err := cm.client.RemSorted(cm.deletion...); err != nil { + if err := cm.client.RemSorted(ctx, cm.deletion...); err != nil { return errors.Trace(err) } } if len(cm.insertion) > 0 { - if err := cm.client.AddSorted(cm.insertion...); err != nil { + if err := cm.client.AddSorted(ctx, cm.insertion...); err != nil { return errors.Trace(err) } } diff --git a/storage/cache/database.go b/storage/cache/database.go index b0191f906..a92be5932 100644 --- a/storage/cache/database.go +++ b/storage/cache/database.go @@ -16,9 +16,15 @@ package cache import ( "context" - "database/sql" + "sort" + "strconv" + "strings" + "time" + + "github.com/XSAM/otelsql" "github.com/araddon/dateparse" "github.com/dzwvip/oracle" + "github.com/go-redis/redis/extra/redisotel/v9" "github.com/go-redis/redis/v9" "github.com/juju/errors" "github.com/samber/lo" @@ -27,16 +33,15 @@ import ( "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/x/mongo/driver/connstring" + "go.opentelemetry.io/contrib/instrumentation/go.mongodb.org/mongo-driver/mongo/otelmongo" + semconv "go.opentelemetry.io/otel/semconv/v1.8.0" + "go.uber.org/zap" "gorm.io/driver/mysql" "gorm.io/driver/postgres" "gorm.io/driver/sqlite" "gorm.io/gorm" "gorm.io/gorm/logger" "moul.io/zapgorm2" - "sort" - "strconv" - "strings" - "time" ) const ( @@ -275,21 +280,21 @@ type Database interface { Scan(work func(string) error) error Purge() error - Set(values ...Value) error - Get(name string) *ReturnValue - Delete(name string) error - - GetSet(key string) ([]string, error) - SetSet(key string, members ...string) error - AddSet(key string, members ...string) error - RemSet(key string, members ...string) error - - AddSorted(sortedSets ...SortedSet) error - GetSorted(key string, begin, end int) ([]Scored, error) - GetSortedByScore(key string, begin, end float64) ([]Scored, error) - RemSortedByScore(key string, begin, end float64) error - SetSorted(key string, scores []Scored) error - RemSorted(members ...SetMember) error + Set(ctx context.Context, values ...Value) error + Get(ctx context.Context, name string) *ReturnValue + Delete(ctx context.Context, name string) error + + GetSet(ctx context.Context, key string) ([]string, error) + SetSet(ctx context.Context, key string, members ...string) error + AddSet(ctx context.Context, key string, members ...string) error + RemSet(ctx context.Context, key string, members ...string) error + + AddSorted(ctx context.Context, sortedSets ...SortedSet) error + GetSorted(ctx context.Context, key string, begin, end int) ([]Scored, error) + GetSortedByScore(ctx context.Context, key string, begin, end float64) ([]Scored, error) + RemSortedByScore(ctx context.Context, key string, begin, end float64) error + SetSorted(ctx context.Context, key string, scores []Scored) error + RemSorted(ctx context.Context, members ...SetMember) error } // Open a connection to a database. @@ -303,6 +308,10 @@ func Open(path, tablePrefix string) (Database, error) { database := new(Redis) database.client = redis.NewClient(opt) database.TablePrefix = storage.TablePrefix(tablePrefix) + if err = redisotel.InstrumentTracing(database.client, redisotel.WithAttributes(semconv.DBSystemRedis)); err != nil { + log.Logger().Error("failed to add tracing for redis", zap.Error(err)) + return nil, errors.Trace(err) + } return database, nil } else if strings.HasPrefix(path, storage.RedisClusterPrefix) { opt, err := ParseRedisClusterURL(path) @@ -312,11 +321,18 @@ func Open(path, tablePrefix string) (Database, error) { database := new(Redis) database.client = redis.NewClusterClient(opt) database.TablePrefix = storage.TablePrefix(tablePrefix) + if err = redisotel.InstrumentTracing(database.client, redisotel.WithAttributes(semconv.DBSystemRedis)); err != nil { + log.Logger().Error("failed to add tracing for redis", zap.Error(err)) + return nil, errors.Trace(err) + } return database, nil } else if strings.HasPrefix(path, storage.MongoPrefix) || strings.HasPrefix(path, storage.MongoSrvPrefix) { // connect to database database := new(MongoDB) - if database.client, err = mongo.Connect(context.Background(), options.Client().ApplyURI(path)); err != nil { + opts := options.Client() + opts.Monitor = otelmongo.NewMonitor() + opts.ApplyURI(path) + if database.client, err = mongo.Connect(context.Background(), opts); err != nil { return nil, errors.Trace(err) } // parse DSN and extract database name @@ -331,7 +347,10 @@ func Open(path, tablePrefix string) (Database, error) { database := new(SQLDatabase) database.driver = Postgres database.TablePrefix = storage.TablePrefix(tablePrefix) - if database.client, err = sql.Open("postgres", path); err != nil { + if database.client, err = otelsql.Open("postgres", path, + otelsql.WithAttributes(semconv.DBSystemPostgreSQL), + otelsql.WithSpanOptions(otelsql.SpanOptions{DisableErrSkip: true}), + ); err != nil { return nil, errors.Trace(err) } database.gormDB, err = gorm.Open(postgres.New(postgres.Config{Conn: database.client}), storage.NewGORMConfig(tablePrefix)) @@ -356,7 +375,10 @@ func Open(path, tablePrefix string) (Database, error) { database := new(SQLDatabase) database.driver = MySQL database.TablePrefix = storage.TablePrefix(tablePrefix) - if database.client, err = sql.Open("mysql", name); err != nil { + if database.client, err = otelsql.Open("mysql", name, + otelsql.WithAttributes(semconv.DBSystemMySQL), + otelsql.WithSpanOptions(otelsql.SpanOptions{DisableErrSkip: true}), + ); err != nil { return nil, errors.Trace(err) } database.gormDB, err = gorm.Open(mysql.New(mysql.Config{Conn: database.client}), storage.NewGORMConfig(tablePrefix)) @@ -377,7 +399,10 @@ func Open(path, tablePrefix string) (Database, error) { database := new(SQLDatabase) database.driver = SQLite database.TablePrefix = storage.TablePrefix(tablePrefix) - if database.client, err = sql.Open("sqlite", name); err != nil { + if database.client, err = otelsql.Open("sqlite", name, + otelsql.WithAttributes(semconv.DBSystemSqlite), + otelsql.WithSpanOptions(otelsql.SpanOptions{DisableErrSkip: true}), + ); err != nil { return nil, errors.Trace(err) } gormConfig := storage.NewGORMConfig(tablePrefix) @@ -397,7 +422,10 @@ func Open(path, tablePrefix string) (Database, error) { database := new(SQLDatabase) database.driver = Oracle database.TablePrefix = storage.TablePrefix(tablePrefix) - if database.client, err = sql.Open("oracle", path); err != nil { + if database.client, err = otelsql.Open("oracle", path, + otelsql.WithAttributes(semconv.DBSystemOracle), + otelsql.WithSpanOptions(otelsql.SpanOptions{DisableErrSkip: true}), + ); err != nil { return nil, errors.Trace(err) } database.gormDB, err = gorm.Open(oracle.New(oracle.Config{Conn: database.client}), storage.NewGORMConfig(tablePrefix)) diff --git a/storage/cache/database_test.go b/storage/cache/database_test.go index 0e928145f..c97f95caf 100644 --- a/storage/cache/database_test.go +++ b/storage/cache/database_test.go @@ -14,6 +14,7 @@ package cache import ( + "context" "github.com/juju/errors" "github.com/stretchr/testify/assert" "math" @@ -22,87 +23,90 @@ import ( ) func testMeta(t *testing.T, db Database) { + ctx := context.Background() // Set meta string - err := db.Set(String(Key("meta", "1"), "2"), String(Key("meta", "1000"), "10")) + err := db.Set(ctx, String(Key("meta", "1"), "2"), String(Key("meta", "1000"), "10")) assert.NoError(t, err) // Get meta string - value, err := db.Get(Key("meta", "1")).String() + value, err := db.Get(ctx, Key("meta", "1")).String() assert.NoError(t, err) assert.Equal(t, "2", value) - value, err = db.Get(Key("meta", "1000")).String() + value, err = db.Get(ctx, Key("meta", "1000")).String() assert.NoError(t, err) assert.Equal(t, "10", value) // Delete string - err = db.Delete(Key("meta", "1")) + err = db.Delete(ctx, Key("meta", "1")) assert.NoError(t, err) // Get meta not existed - value, err = db.Get(Key("meta", "1")).String() + value, err = db.Get(ctx, Key("meta", "1")).String() assert.True(t, errors.Is(err, errors.NotFound), err) assert.Equal(t, "", value) // Set meta int - err = db.Set(Integer(Key("meta", "1"), 2)) + err = db.Set(ctx, Integer(Key("meta", "1"), 2)) assert.NoError(t, err) // Get meta int - valInt, err := db.Get(Key("meta", "1")).Integer() + valInt, err := db.Get(ctx, Key("meta", "1")).Integer() assert.NoError(t, err) assert.Equal(t, 2, valInt) // set meta time - err = db.Set(Time(Key("meta", "1"), time.Date(1996, 4, 8, 0, 0, 0, 0, time.UTC))) + err = db.Set(ctx, Time(Key("meta", "1"), time.Date(1996, 4, 8, 0, 0, 0, 0, time.UTC))) assert.NoError(t, err) // get meta time - valTime, err := db.Get(Key("meta", "1")).Time() + valTime, err := db.Get(ctx, Key("meta", "1")).Time() assert.NoError(t, err) assert.Equal(t, 1996, valTime.Year()) assert.Equal(t, time.Month(4), valTime.Month()) assert.Equal(t, 8, valTime.Day()) // test set empty - err = db.Set() + err = db.Set(ctx) assert.NoError(t, err) // test set duplicate - err = db.Set(String("100", "1"), String("100", "2")) + err = db.Set(ctx, String("100", "1"), String("100", "2")) assert.NoError(t, err) } func testSet(t *testing.T, db Database) { - err := db.SetSet("set", "1") + ctx := context.Background() + err := db.SetSet(ctx, "set", "1") assert.NoError(t, err) // test add - err = db.AddSet("set", "2") + err = db.AddSet(ctx, "set", "2") assert.NoError(t, err) var members []string - members, err = db.GetSet("set") + members, err = db.GetSet(ctx, "set") assert.NoError(t, err) assert.Equal(t, []string{"1", "2"}, members) // test rem - err = db.RemSet("set", "1") + err = db.RemSet(ctx, "set", "1") assert.NoError(t, err) - members, err = db.GetSet("set") + members, err = db.GetSet(ctx, "set") assert.NoError(t, err) assert.Equal(t, []string{"2"}, members) // test set - err = db.SetSet("set", "3") + err = db.SetSet(ctx, "set", "3") assert.NoError(t, err) - members, err = db.GetSet("set") + members, err = db.GetSet(ctx, "set") assert.NoError(t, err) assert.Equal(t, []string{"3"}, members) // test add empty - err = db.AddSet("set") + err = db.AddSet(ctx, "set") assert.NoError(t, err) // test set empty - err = db.SetSet("set") + err = db.SetSet(ctx, "set") assert.NoError(t, err) // test get empty - members, err = db.GetSet("unknown_set") + members, err = db.GetSet(ctx, "unknown_set") assert.NoError(t, err) assert.Empty(t, members) // test rem empty - err = db.RemSet("set") + err = db.RemSet(ctx, "set") assert.NoError(t, err) } func testSort(t *testing.T, db Database) { + ctx := context.Background() // Put scores scores := []Scored{ {"0", 0}, @@ -111,12 +115,12 @@ func testSort(t *testing.T, db Database) { {"3", 1.3}, {"4", 1.4}, } - err := db.SetSorted("sort", scores[:3]) + err := db.SetSorted(ctx, "sort", scores[:3]) assert.NoError(t, err) - err = db.AddSorted(SortedSet{"sort", scores[3:]}) + err = db.AddSorted(ctx, SortedSet{"sort", scores[3:]}) assert.NoError(t, err) // Get scores - totalItems, err := db.GetSorted("sort", 0, -1) + totalItems, err := db.GetSorted(ctx, "sort", 0, -1) assert.NoError(t, err) assert.Equal(t, []Scored{ {"4", 1.4}, @@ -125,21 +129,21 @@ func testSort(t *testing.T, db Database) { {"1", 1.1}, {"0", 0}, }, totalItems) - halfItems, err := db.GetSorted("sort", 0, 2) + halfItems, err := db.GetSorted(ctx, "sort", 0, 2) assert.NoError(t, err) assert.Equal(t, []Scored{ {"4", 1.4}, {"3", 1.3}, {"2", 1.2}, }, halfItems) - halfItems, err = db.GetSorted("sort", 1, 2) + halfItems, err = db.GetSorted(ctx, "sort", 1, 2) assert.NoError(t, err) assert.Equal(t, []Scored{ {"3", 1.3}, {"2", 1.2}, }, halfItems) // get scores by score - partItems, err := db.GetSortedByScore("sort", 1.1, 1.3) + partItems, err := db.GetSortedByScore(ctx, "sort", 1.1, 1.3) assert.NoError(t, err) assert.Equal(t, []Scored{ {"1", 1.1}, @@ -147,20 +151,20 @@ func testSort(t *testing.T, db Database) { {"3", 1.3}, }, partItems) // remove scores by score - err = db.AddSorted(SortedSet{"sort", []Scored{ + err = db.AddSorted(ctx, SortedSet{"sort", []Scored{ {"5", -5}, {"6", -6}, }}) assert.NoError(t, err) - err = db.RemSortedByScore("sort", math.Inf(-1), -1) + err = db.RemSortedByScore(ctx, "sort", math.Inf(-1), -1) assert.NoError(t, err) - partItems, err = db.GetSortedByScore("sort", math.Inf(-1), -1) + partItems, err = db.GetSortedByScore(ctx, "sort", math.Inf(-1), -1) assert.NoError(t, err) assert.Empty(t, partItems) // Remove score - err = db.RemSorted(Member("sort", "0")) + err = db.RemSorted(ctx, Member("sort", "0")) assert.NoError(t, err) - totalItems, err = db.GetSorted("sort", 0, -1) + totalItems, err = db.GetSorted(ctx, "sort", 0, -1) assert.NoError(t, err) assert.Equal(t, []Scored{ {"4", 1.4}, @@ -170,37 +174,38 @@ func testSort(t *testing.T, db Database) { }, totalItems) // test set empty - err = db.SetSorted("sort", []Scored{}) + err = db.SetSorted(ctx, "sort", []Scored{}) assert.NoError(t, err) // test set duplicate - err = db.SetSorted("sort1000", []Scored{ + err = db.SetSorted(ctx, "sort1000", []Scored{ {"100", 1}, {"100", 2}, }) assert.NoError(t, err) // test add empty - err = db.AddSorted() + err = db.AddSorted(ctx) assert.NoError(t, err) - err = db.AddSorted(SortedSet{}) + err = db.AddSorted(ctx, SortedSet{}) assert.NoError(t, err) // test add duplicate - err = db.AddSorted(SortedSet{"sort1000", []Scored{{"100", 1}, {"100", 2}}}) + err = db.AddSorted(ctx, SortedSet{"sort1000", []Scored{{"100", 1}, {"100", 2}}}) assert.NoError(t, err) // test get empty - scores, err = db.GetSorted("sort", 0, -1) + scores, err = db.GetSorted(ctx, "sort", 0, -1) assert.NoError(t, err) assert.Empty(t, scores) - scores, err = db.GetSorted("sort", 10, 5) + scores, err = db.GetSorted(ctx, "sort", 10, 5) assert.NoError(t, err) assert.Empty(t, scores) } func testScan(t *testing.T, db Database) { - err := db.Set(String("1", "1")) + ctx := context.Background() + err := db.Set(ctx, String("1", "1")) assert.NoError(t, err) - err = db.SetSet("2", "21", "22", "23") + err = db.SetSet(ctx, "2", "21", "22", "23") assert.NoError(t, err) - err = db.SetSorted("3", []Scored{{"1", 1}, {"2", 2}, {"3", 3}}) + err = db.SetSorted(ctx, "3", []Scored{{"1", 1}, {"2", 2}, {"3", 3}}) assert.NoError(t, err) var keys []string @@ -213,34 +218,35 @@ func testScan(t *testing.T, db Database) { } func testPurge(t *testing.T, db Database) { + ctx := context.Background() // insert data - err := db.Set(String("key", "value")) + err := db.Set(ctx, String("key", "value")) assert.NoError(t, err) - ret := db.Get("key") + ret := db.Get(ctx, "key") assert.NoError(t, ret.err) assert.Equal(t, "value", ret.value) - err = db.AddSet("set", "a", "b", "c") + err = db.AddSet(ctx, "set", "a", "b", "c") assert.NoError(t, err) - s, err := db.GetSet("set") + s, err := db.GetSet(ctx, "set") assert.NoError(t, err) assert.ElementsMatch(t, []string{"a", "b", "c"}, s) - err = db.AddSorted(Sorted("sorted", []Scored{{Id: "a", Score: 1}, {Id: "b", Score: 2}, {Id: "c", Score: 3}})) + err = db.AddSorted(ctx, Sorted("sorted", []Scored{{Id: "a", Score: 1}, {Id: "b", Score: 2}, {Id: "c", Score: 3}})) assert.NoError(t, err) - z, err := db.GetSorted("sorted", 0, -1) + z, err := db.GetSorted(ctx, "sorted", 0, -1) assert.NoError(t, err) assert.ElementsMatch(t, []Scored{{Id: "a", Score: 1}, {Id: "b", Score: 2}, {Id: "c", Score: 3}}, z) // purge data err = db.Purge() assert.NoError(t, err) - ret = db.Get("key") + ret = db.Get(ctx, "key") assert.ErrorIs(t, ret.err, errors.NotFound) - s, err = db.GetSet("set") + s, err = db.GetSet(ctx, "set") assert.NoError(t, err) assert.Empty(t, s) - z, err = db.GetSorted("sorted", 0, -1) + z, err = db.GetSorted(ctx, "sorted", 0, -1) assert.NoError(t, err) assert.Empty(t, z) diff --git a/storage/cache/mongodb.go b/storage/cache/mongodb.go index 84bfb26dc..0d2807ce8 100644 --- a/storage/cache/mongodb.go +++ b/storage/cache/mongodb.go @@ -179,11 +179,10 @@ func (m MongoDB) Purge() error { return nil } -func (m MongoDB) Set(values ...Value) error { +func (m MongoDB) Set(ctx context.Context, values ...Value) error { if len(values) == 0 { return nil } - ctx := context.Background() c := m.client.Database(m.dbName).Collection(m.ValuesTable()) var models []mongo.WriteModel for _, value := range values { @@ -196,8 +195,7 @@ func (m MongoDB) Set(values ...Value) error { return errors.Trace(err) } -func (m MongoDB) Get(name string) *ReturnValue { - ctx := context.Background() +func (m MongoDB) Get(ctx context.Context, name string) *ReturnValue { c := m.client.Database(m.dbName).Collection(m.ValuesTable()) r := c.FindOne(ctx, bson.M{"_id": bson.M{"$eq": name}}) if err := r.Err(); err == mongo.ErrNoDocuments { @@ -212,15 +210,13 @@ func (m MongoDB) Get(name string) *ReturnValue { } } -func (m MongoDB) Delete(name string) error { - ctx := context.Background() +func (m MongoDB) Delete(ctx context.Context, name string) error { c := m.client.Database(m.dbName).Collection(m.ValuesTable()) _, err := c.DeleteOne(ctx, bson.M{"_id": bson.M{"$eq": name}}) return errors.Trace(err) } -func (m MongoDB) GetSet(name string) ([]string, error) { - ctx := context.Background() +func (m MongoDB) GetSet(ctx context.Context, name string) ([]string, error) { c := m.client.Database(m.dbName).Collection(m.SetsTable()) r, err := c.Find(ctx, bson.M{"name": name}) if err != nil { @@ -237,8 +233,7 @@ func (m MongoDB) GetSet(name string) ([]string, error) { return members, nil } -func (m MongoDB) SetSet(name string, members ...string) error { - ctx := context.Background() +func (m MongoDB) SetSet(ctx context.Context, name string, members ...string) error { c := m.client.Database(m.dbName).Collection(m.SetsTable()) var models []mongo.WriteModel models = append(models, mongo.NewDeleteManyModel().SetFilter(bson.M{"name": bson.M{"$eq": name}})) @@ -252,11 +247,10 @@ func (m MongoDB) SetSet(name string, members ...string) error { return errors.Trace(err) } -func (m MongoDB) AddSet(name string, members ...string) error { +func (m MongoDB) AddSet(ctx context.Context, name string, members ...string) error { if len(members) == 0 { return nil } - ctx := context.Background() c := m.client.Database(m.dbName).Collection(m.SetsTable()) var models []mongo.WriteModel for _, member := range members { @@ -269,11 +263,10 @@ func (m MongoDB) AddSet(name string, members ...string) error { return errors.Trace(err) } -func (m MongoDB) RemSet(name string, members ...string) error { +func (m MongoDB) RemSet(ctx context.Context, name string, members ...string) error { if len(members) == 0 { return nil } - ctx := context.Background() c := m.client.Database(m.dbName).Collection(m.SetsTable()) var models []mongo.WriteModel for _, member := range members { @@ -284,8 +277,7 @@ func (m MongoDB) RemSet(name string, members ...string) error { return errors.Trace(err) } -func (m MongoDB) GetSorted(name string, begin, end int) ([]Scored, error) { - ctx := context.Background() +func (m MongoDB) GetSorted(ctx context.Context, name string, begin, end int) ([]Scored, error) { c := m.client.Database(m.dbName).Collection(m.SortedSetsTable()) opt := options.Find() opt.SetSort(bson.M{"score": -1}) @@ -313,8 +305,7 @@ func (m MongoDB) GetSorted(name string, begin, end int) ([]Scored, error) { return scores, nil } -func (m MongoDB) GetSortedByScore(name string, begin, end float64) ([]Scored, error) { - ctx := context.Background() +func (m MongoDB) GetSortedByScore(ctx context.Context, name string, begin, end float64) ([]Scored, error) { c := m.client.Database(m.dbName).Collection(m.SortedSetsTable()) opt := options.Find() opt.SetSort(bson.M{"score": 1}) @@ -340,8 +331,7 @@ func (m MongoDB) GetSortedByScore(name string, begin, end float64) ([]Scored, er return scores, nil } -func (m MongoDB) RemSortedByScore(name string, begin, end float64) error { - ctx := context.Background() +func (m MongoDB) RemSortedByScore(ctx context.Context, name string, begin, end float64) error { c := m.client.Database(m.dbName).Collection(m.SortedSetsTable()) _, err := c.DeleteMany(ctx, bson.D{ {"name", name}, @@ -351,8 +341,7 @@ func (m MongoDB) RemSortedByScore(name string, begin, end float64) error { return errors.Trace(err) } -func (m MongoDB) AddSorted(sortedSets ...SortedSet) error { - ctx := context.Background() +func (m MongoDB) AddSorted(ctx context.Context, sortedSets ...SortedSet) error { c := m.client.Database(m.dbName).Collection(m.SortedSetsTable()) var models []mongo.WriteModel for _, sorted := range sortedSets { @@ -370,8 +359,7 @@ func (m MongoDB) AddSorted(sortedSets ...SortedSet) error { return errors.Trace(err) } -func (m MongoDB) SetSorted(name string, scores []Scored) error { - ctx := context.Background() +func (m MongoDB) SetSorted(ctx context.Context, name string, scores []Scored) error { c := m.client.Database(m.dbName).Collection(m.SortedSetsTable()) var models []mongo.WriteModel models = append(models, mongo.NewDeleteManyModel().SetFilter(bson.M{"name": bson.M{"$eq": name}})) @@ -385,11 +373,10 @@ func (m MongoDB) SetSorted(name string, scores []Scored) error { return errors.Trace(err) } -func (m MongoDB) RemSorted(members ...SetMember) error { +func (m MongoDB) RemSorted(ctx context.Context, members ...SetMember) error { if len(members) == 0 { return nil } - ctx := context.Background() c := m.client.Database(m.dbName).Collection(m.SortedSetsTable()) var models []mongo.WriteModel for _, member := range members { diff --git a/storage/cache/no_database.go b/storage/cache/no_database.go index b207cf333..de0b50685 100644 --- a/storage/cache/no_database.go +++ b/storage/cache/no_database.go @@ -14,6 +14,8 @@ package cache +import "context" + // NoDatabase means no database used for cache. type NoDatabase struct{} @@ -35,66 +37,66 @@ func (NoDatabase) Purge() error { return ErrNoDatabase } -func (NoDatabase) Set(_ ...Value) error { +func (NoDatabase) Set(_ context.Context, _ ...Value) error { return ErrNoDatabase } // Get method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) Get(_ string) *ReturnValue { +func (NoDatabase) Get(_ context.Context, _ string) *ReturnValue { return &ReturnValue{err: ErrNoDatabase} } // Delete method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) Delete(_ string) error { +func (NoDatabase) Delete(_ context.Context, _ string) error { return ErrNoDatabase } // GetSet method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) GetSet(_ string) ([]string, error) { +func (NoDatabase) GetSet(_ context.Context, _ string) ([]string, error) { return nil, ErrNoDatabase } // SetSet method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) SetSet(_ string, _ ...string) error { +func (NoDatabase) SetSet(_ context.Context, _ string, _ ...string) error { return ErrNoDatabase } // AddSet method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) AddSet(_ string, _ ...string) error { +func (NoDatabase) AddSet(_ context.Context, _ string, _ ...string) error { return ErrNoDatabase } // RemSet method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) RemSet(_ string, _ ...string) error { +func (NoDatabase) RemSet(_ context.Context, _ string, _ ...string) error { return ErrNoDatabase } // GetSorted method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) GetSorted(_ string, _, _ int) ([]Scored, error) { +func (NoDatabase) GetSorted(_ context.Context, _ string, _, _ int) ([]Scored, error) { return nil, ErrNoDatabase } // GetSortedByScore method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) GetSortedByScore(_ string, _, _ float64) ([]Scored, error) { +func (NoDatabase) GetSortedByScore(_ context.Context, _ string, _, _ float64) ([]Scored, error) { return nil, ErrNoDatabase } // RemSortedByScore method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) RemSortedByScore(_ string, _, _ float64) error { +func (NoDatabase) RemSortedByScore(_ context.Context, _ string, _, _ float64) error { return ErrNoDatabase } // AddSorted method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) AddSorted(_ ...SortedSet) error { +func (NoDatabase) AddSorted(_ context.Context, _ ...SortedSet) error { return ErrNoDatabase } // SetSorted method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) SetSorted(_ string, _ []Scored) error { +func (NoDatabase) SetSorted(_ context.Context, _ string, _ []Scored) error { return ErrNoDatabase } // RemSorted method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) RemSorted(_ ...SetMember) error { +func (NoDatabase) RemSorted(_ context.Context, _ ...SetMember) error { return ErrNoDatabase } diff --git a/storage/cache/no_database_test.go b/storage/cache/no_database_test.go index d10bf092f..21840eca9 100644 --- a/storage/cache/no_database_test.go +++ b/storage/cache/no_database_test.go @@ -15,11 +15,13 @@ package cache import ( + "context" "github.com/stretchr/testify/assert" "testing" ) func TestNoDatabase(t *testing.T) { + ctx := context.Background() var database NoDatabase err := database.Close() assert.ErrorIs(t, err, ErrNoDatabase) @@ -29,36 +31,36 @@ func TestNoDatabase(t *testing.T) { assert.ErrorIs(t, err, ErrNoDatabase) err = database.Purge() assert.ErrorIs(t, err, ErrNoDatabase) - err = database.Set() + err = database.Set(ctx) assert.ErrorIs(t, err, ErrNoDatabase) - _, err = database.Get(Key("", "")).String() + _, err = database.Get(ctx, Key("", "")).String() assert.ErrorIs(t, err, ErrNoDatabase) - _, err = database.Get(Key("", "")).Integer() + _, err = database.Get(ctx, Key("", "")).Integer() assert.ErrorIs(t, err, ErrNoDatabase) - _, err = database.Get(Key("", "")).Time() + _, err = database.Get(ctx, Key("", "")).Time() assert.ErrorIs(t, err, ErrNoDatabase) - err = database.Delete(Key("", "")) + err = database.Delete(ctx, Key("", "")) assert.ErrorIs(t, err, ErrNoDatabase) - _, err = database.GetSet("") + _, err = database.GetSet(ctx, "") assert.ErrorIs(t, err, ErrNoDatabase) - err = database.SetSet("") + err = database.SetSet(ctx, "") assert.ErrorIs(t, err, ErrNoDatabase) - err = database.AddSet("") + err = database.AddSet(ctx, "") assert.ErrorIs(t, err, ErrNoDatabase) - err = database.RemSet("", "") + err = database.RemSet(ctx, "", "") assert.ErrorIs(t, err, ErrNoDatabase) - _, err = database.GetSorted("", 0, 0) + _, err = database.GetSorted(ctx, "", 0, 0) assert.ErrorIs(t, err, ErrNoDatabase) - _, err = database.GetSortedByScore("", 0, 0) + _, err = database.GetSortedByScore(ctx, "", 0, 0) assert.ErrorIs(t, err, ErrNoDatabase) - err = database.RemSortedByScore("", 0, 0) + err = database.RemSortedByScore(ctx, "", 0, 0) assert.ErrorIs(t, err, ErrNoDatabase) - err = database.SetSorted("", nil) + err = database.SetSorted(ctx, "", nil) assert.ErrorIs(t, err, ErrNoDatabase) - err = database.AddSorted() + err = database.AddSorted(ctx) assert.ErrorIs(t, err, ErrNoDatabase) - err = database.RemSorted() + err = database.RemSorted(ctx) assert.ErrorIs(t, err, ErrNoDatabase) } diff --git a/storage/cache/redis.go b/storage/cache/redis.go index eda3cce83..fbe2e9cae 100644 --- a/storage/cache/redis.go +++ b/storage/cache/redis.go @@ -276,8 +276,7 @@ func (r *Redis) purge(ctx context.Context, client redis.UniversalClient, isClust } } -func (r *Redis) Set(values ...Value) error { - var ctx = context.Background() +func (r *Redis) Set(ctx context.Context, values ...Value) error { p := r.client.Pipeline() for _, v := range values { if err := p.Set(ctx, r.Key(v.name), v.value, 0).Err(); err != nil { @@ -289,8 +288,7 @@ func (r *Redis) Set(values ...Value) error { } // Get returns a value from Redis. -func (r *Redis) Get(key string) *ReturnValue { - var ctx = context.Background() +func (r *Redis) Get(ctx context.Context, key string) *ReturnValue { val, err := r.client.Get(ctx, r.Key(key)).Result() if err != nil { if err == redis.Nil { @@ -302,19 +300,17 @@ func (r *Redis) Get(key string) *ReturnValue { } // Delete object from Redis. -func (r *Redis) Delete(key string) error { - ctx := context.Background() +func (r *Redis) Delete(ctx context.Context, key string) error { return r.client.Del(ctx, r.Key(key)).Err() } // GetSet returns members of a set from Redis. -func (r *Redis) GetSet(key string) ([]string, error) { - ctx := context.Background() +func (r *Redis) GetSet(ctx context.Context, key string) ([]string, error) { return r.client.SMembers(ctx, r.Key(key)).Result() } // SetSet overrides a set with members in Redis. -func (r *Redis) SetSet(key string, members ...string) error { +func (r *Redis) SetSet(ctx context.Context, key string, members ...string) error { if len(members) == 0 { return nil } @@ -324,7 +320,6 @@ func (r *Redis) SetSet(key string, members ...string) error { values = append(values, member) } // push set - ctx := context.Background() pipeline := r.client.Pipeline() pipeline.Del(ctx, r.Key(key)) pipeline.SAdd(ctx, r.Key(key), values...) @@ -333,7 +328,7 @@ func (r *Redis) SetSet(key string, members ...string) error { } // AddSet adds members to a set in Redis. -func (r *Redis) AddSet(key string, members ...string) error { +func (r *Redis) AddSet(ctx context.Context, key string, members ...string) error { if len(members) == 0 { return nil } @@ -343,22 +338,19 @@ func (r *Redis) AddSet(key string, members ...string) error { values = append(values, member) } // push set - ctx := context.Background() return r.client.SAdd(ctx, r.Key(key), values...).Err() } // RemSet removes members from a set in Redis. -func (r *Redis) RemSet(key string, members ...string) error { +func (r *Redis) RemSet(ctx context.Context, key string, members ...string) error { if len(members) == 0 { return nil } - ctx := context.Background() return r.client.SRem(ctx, r.Key(key), members).Err() } // GetSorted get scores from sorted set. -func (r *Redis) GetSorted(key string, begin, end int) ([]Scored, error) { - ctx := context.Background() +func (r *Redis) GetSorted(ctx context.Context, key string, begin, end int) ([]Scored, error) { members, err := r.client.ZRevRangeWithScores(ctx, r.Key(key), int64(begin), int64(end)).Result() if err != nil { return nil, err @@ -370,8 +362,7 @@ func (r *Redis) GetSorted(key string, begin, end int) ([]Scored, error) { return results, nil } -func (r *Redis) GetSortedByScore(key string, begin, end float64) ([]Scored, error) { - ctx := context.Background() +func (r *Redis) GetSortedByScore(ctx context.Context, key string, begin, end float64) ([]Scored, error) { members, err := r.client.ZRangeByScoreWithScores(ctx, r.Key(key), &redis.ZRangeBy{ Min: strconv.FormatFloat(begin, 'g', -1, 64), Max: strconv.FormatFloat(end, 'g', -1, 64), @@ -388,8 +379,7 @@ func (r *Redis) GetSortedByScore(key string, begin, end float64) ([]Scored, erro return results, nil } -func (r *Redis) RemSortedByScore(key string, begin, end float64) error { - ctx := context.Background() +func (r *Redis) RemSortedByScore(ctx context.Context, key string, begin, end float64) error { return r.client.ZRemRangeByScore(ctx, r.Key(key), strconv.FormatFloat(begin, 'g', -1, 64), strconv.FormatFloat(end, 'g', -1, 64)). @@ -397,8 +387,7 @@ func (r *Redis) RemSortedByScore(key string, begin, end float64) error { } // AddSorted add scores to sorted set. -func (r *Redis) AddSorted(sortedSets ...SortedSet) error { - ctx := context.Background() +func (r *Redis) AddSorted(ctx context.Context, sortedSets ...SortedSet) error { p := r.client.Pipeline() for _, sorted := range sortedSets { if len(sorted.scores) > 0 { @@ -414,12 +403,11 @@ func (r *Redis) AddSorted(sortedSets ...SortedSet) error { } // SetSorted set scores in sorted set and clear previous scores. -func (r *Redis) SetSorted(key string, scores []Scored) error { +func (r *Redis) SetSorted(ctx context.Context, key string, scores []Scored) error { members := make([]redis.Z, 0, len(scores)) for _, score := range scores { members = append(members, redis.Z{Member: score.Id, Score: float64(score.Score)}) } - ctx := context.Background() pipeline := r.client.Pipeline() pipeline.Del(ctx, r.Key(key)) if len(scores) > 0 { @@ -430,11 +418,10 @@ func (r *Redis) SetSorted(key string, scores []Scored) error { } // RemSorted method of NoDatabase returns ErrNoDatabase. -func (r *Redis) RemSorted(members ...SetMember) error { +func (r *Redis) RemSorted(ctx context.Context, members ...SetMember) error { if len(members) == 0 { return nil } - ctx := context.Background() pipe := r.client.Pipeline() for _, member := range members { pipe.ZRem(ctx, r.Key(member.name), member.member) diff --git a/storage/cache/sql.go b/storage/cache/sql.go index 0c1fd43df..8c182a2c3 100644 --- a/storage/cache/sql.go +++ b/storage/cache/sql.go @@ -15,6 +15,7 @@ package cache import ( + "context" "database/sql" "fmt" _ "github.com/go-sql-driver/mysql" @@ -148,7 +149,7 @@ func (db *SQLDatabase) Purge() error { return nil } -func (db *SQLDatabase) Set(values ...Value) error { +func (db *SQLDatabase) Set(ctx context.Context, values ...Value) error { if len(values) == 0 { return nil } @@ -163,15 +164,15 @@ func (db *SQLDatabase) Set(values ...Value) error { valueSet.Add(value.name) } } - err := db.gormDB.Clauses(clause.OnConflict{ + err := db.gormDB.WithContext(ctx).Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "name"}}, DoUpdates: clause.AssignmentColumns([]string{"value"}), }).Create(rows).Error return errors.Trace(err) } -func (db *SQLDatabase) Get(name string) *ReturnValue { - rs, err := db.gormDB.Table(db.ValuesTable()).Where("name = ?", name).Select("value").Rows() +func (db *SQLDatabase) Get(ctx context.Context, name string) *ReturnValue { + rs, err := db.gormDB.WithContext(ctx).Table(db.ValuesTable()).Where("name = ?", name).Select("value").Rows() if err != nil { return &ReturnValue{err: errors.Trace(err)} } @@ -187,13 +188,13 @@ func (db *SQLDatabase) Get(name string) *ReturnValue { return &ReturnValue{err: errors.Annotate(ErrObjectNotExist, name)} } -func (db *SQLDatabase) Delete(name string) error { - err := db.gormDB.Delete(&SQLValue{Name: name}).Error +func (db *SQLDatabase) Delete(ctx context.Context, name string) error { + err := db.gormDB.WithContext(ctx).Delete(&SQLValue{Name: name}).Error return errors.Trace(err) } -func (db *SQLDatabase) GetSet(key string) ([]string, error) { - rs, err := db.gormDB.Table(db.SetsTable()).Select("member").Where("name = ?", key).Rows() +func (db *SQLDatabase) GetSet(ctx context.Context, key string) ([]string, error) { + rs, err := db.gormDB.WithContext(ctx).Table(db.SetsTable()).Select("member").Where("name = ?", key).Rows() if err != nil { return nil, errors.Trace(err) } @@ -209,8 +210,9 @@ func (db *SQLDatabase) GetSet(key string) ([]string, error) { return members, nil } -func (db *SQLDatabase) SetSet(key string, members ...string) error { - err := db.gormDB.Delete(&SQLSet{}, "name = ?", key).Error +func (db *SQLDatabase) SetSet(ctx context.Context, key string, members ...string) error { + tx := db.gormDB.WithContext(ctx) + err := tx.Delete(&SQLSet{}, "name = ?", key).Error if err != nil { return errors.Trace(err) } @@ -223,11 +225,11 @@ func (db *SQLDatabase) SetSet(key string, members ...string) error { Member: member, } }) - err = db.gormDB.Clauses(clause.OnConflict{DoNothing: true}).Create(rows).Error + err = tx.Clauses(clause.OnConflict{DoNothing: true}).Create(rows).Error return errors.Trace(err) } -func (db *SQLDatabase) AddSet(key string, members ...string) error { +func (db *SQLDatabase) AddSet(ctx context.Context, key string, members ...string) error { if len(members) == 0 { return nil } @@ -237,11 +239,11 @@ func (db *SQLDatabase) AddSet(key string, members ...string) error { Member: member, } }) - err := db.gormDB.Clauses(clause.OnConflict{DoNothing: true}).Create(rows).Error + err := db.gormDB.WithContext(ctx).Clauses(clause.OnConflict{DoNothing: true}).Create(rows).Error return errors.Trace(err) } -func (db *SQLDatabase) RemSet(key string, members ...string) error { +func (db *SQLDatabase) RemSet(ctx context.Context, key string, members ...string) error { if len(members) == 0 { return nil } @@ -251,11 +253,11 @@ func (db *SQLDatabase) RemSet(key string, members ...string) error { Member: member, } }) - err := db.gormDB.Delete(rows).Error + err := db.gormDB.WithContext(ctx).Delete(rows).Error return errors.Trace(err) } -func (db *SQLDatabase) AddSorted(sortedSets ...SortedSet) error { +func (db *SQLDatabase) AddSorted(ctx context.Context, sortedSets ...SortedSet) error { rows := make([]SQLSortedSet, 0, len(sortedSets)) memberSets := make(map[lo.Tuple2[string, string]]struct{}) for _, sortedSet := range sortedSets { @@ -273,7 +275,7 @@ func (db *SQLDatabase) AddSorted(sortedSets ...SortedSet) error { if len(rows) == 0 { return nil } - if err := db.gormDB.Clauses(clause.OnConflict{ + if err := db.gormDB.WithContext(ctx).Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "name"}, {Name: "member"}}, DoUpdates: clause.AssignmentColumns([]string{"score"}), }).Create(rows).Error; err != nil { @@ -282,8 +284,11 @@ func (db *SQLDatabase) AddSorted(sortedSets ...SortedSet) error { return nil } -func (db *SQLDatabase) GetSorted(key string, begin, end int) ([]Scored, error) { - tx := db.gormDB.Table(db.SortedSetsTable()).Select("member, score").Where("name = ?", key).Order("score DESC") +func (db *SQLDatabase) GetSorted(ctx context.Context, key string, begin, end int) ([]Scored, error) { + tx := db.gormDB.WithContext(ctx).Table(db.SortedSetsTable()). + Select("member, score"). + Where("name = ?", key). + Order("score DESC") if end < begin { tx.Offset(begin).Limit(math.MaxInt64) } else { @@ -305,8 +310,8 @@ func (db *SQLDatabase) GetSorted(key string, begin, end int) ([]Scored, error) { return members, nil } -func (db *SQLDatabase) GetSortedByScore(key string, begin, end float64) ([]Scored, error) { - rs, err := db.gormDB.Table(db.SortedSetsTable()). +func (db *SQLDatabase) GetSortedByScore(ctx context.Context, key string, begin, end float64) ([]Scored, error) { + rs, err := db.gormDB.WithContext(ctx).Table(db.SortedSetsTable()). Select("member, score"). Where("name = ? AND score >= ? AND score <= ?", key, begin, end). Order("score").Rows() @@ -325,13 +330,14 @@ func (db *SQLDatabase) GetSortedByScore(key string, begin, end float64) ([]Score return members, nil } -func (db *SQLDatabase) RemSortedByScore(key string, begin, end float64) error { - err := db.gormDB.Delete(&SQLSortedSet{}, "name = ? AND ? <= score AND score <= ?", key, begin, end).Error +func (db *SQLDatabase) RemSortedByScore(ctx context.Context, key string, begin, end float64) error { + err := db.gormDB.WithContext(ctx).Delete(&SQLSortedSet{}, "name = ? AND ? <= score AND score <= ?", key, begin, end).Error return errors.Trace(err) } -func (db *SQLDatabase) SetSorted(key string, scores []Scored) error { - err := db.gormDB.Delete(&SQLSortedSet{}, "name = ?", key).Error +func (db *SQLDatabase) SetSorted(ctx context.Context, key string, scores []Scored) error { + tx := db.gormDB.WithContext(ctx) + err := tx.Delete(&SQLSortedSet{}, "name = ?", key).Error if err != nil { return errors.Trace(err) } @@ -348,7 +354,7 @@ func (db *SQLDatabase) SetSorted(key string, scores []Scored) error { memberSets[lo.Tuple2[string, string]{key, member.Id}] = struct{}{} } } - err = db.gormDB.Clauses(clause.OnConflict{ + err = tx.Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "name"}, {Name: "member"}}, DoUpdates: clause.AssignmentColumns([]string{"score"}), }).Create(&rows).Error @@ -359,7 +365,7 @@ func (db *SQLDatabase) SetSorted(key string, scores []Scored) error { return nil } -func (db *SQLDatabase) RemSorted(members ...SetMember) error { +func (db *SQLDatabase) RemSorted(ctx context.Context, members ...SetMember) error { if len(members) == 0 { return nil } @@ -369,6 +375,6 @@ func (db *SQLDatabase) RemSorted(members ...SetMember) error { Member: member.member, } }) - err := db.gormDB.Delete(rows).Error + err := db.gormDB.WithContext(ctx).Delete(rows).Error return errors.Trace(err) } diff --git a/storage/data/database.go b/storage/data/database.go index d3cc4d7f3..f045e38aa 100644 --- a/storage/data/database.go +++ b/storage/data/database.go @@ -16,7 +16,12 @@ package data import ( "context" - "database/sql" + "net/url" + "sort" + "strings" + "time" + + "github.com/XSAM/otelsql" "github.com/dzwvip/oracle" "github.com/go-redis/redis/v9" "github.com/juju/errors" @@ -26,6 +31,8 @@ import ( "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "go.mongodb.org/mongo-driver/x/mongo/driver/connstring" + "go.opentelemetry.io/contrib/instrumentation/go.mongodb.org/mongo-driver/mongo/otelmongo" + semconv "go.opentelemetry.io/otel/semconv/v1.12.0" "gorm.io/driver/clickhouse" "gorm.io/driver/mysql" "gorm.io/driver/postgres" @@ -33,10 +40,6 @@ import ( "gorm.io/gorm" "gorm.io/gorm/logger" "moul.io/zapgorm2" - "net/url" - "sort" - "strings" - "time" ) var ( @@ -117,26 +120,26 @@ type Database interface { Close() error Optimize() error Purge() error - BatchInsertItems(items []Item) error - BatchGetItems(itemIds []string) ([]Item, error) - DeleteItem(itemId string) error - GetItem(itemId string) (Item, error) - ModifyItem(itemId string, patch ItemPatch) error - GetItems(cursor string, n int, beginTime *time.Time) (string, []Item, error) - GetItemFeedback(itemId string, feedbackTypes ...string) ([]Feedback, error) - BatchInsertUsers(users []User) error - DeleteUser(userId string) error - GetUser(userId string) (User, error) - ModifyUser(userId string, patch UserPatch) error - GetUsers(cursor string, n int) (string, []User, error) - GetUserFeedback(userId string, endTime *time.Time, feedbackTypes ...string) ([]Feedback, error) - GetUserItemFeedback(userId, itemId string, feedbackTypes ...string) ([]Feedback, error) - DeleteUserItemFeedback(userId, itemId string, feedbackTypes ...string) (int, error) - BatchInsertFeedback(feedback []Feedback, insertUser, insertItem, overwrite bool) error - GetFeedback(cursor string, n int, beginTime, endTime *time.Time, feedbackTypes ...string) (string, []Feedback, error) - GetUserStream(batchSize int) (chan []User, chan error) - GetItemStream(batchSize int, timeLimit *time.Time) (chan []Item, chan error) - GetFeedbackStream(batchSize int, beginTime, endTime *time.Time, feedbackTypes ...string) (chan []Feedback, chan error) + BatchInsertItems(ctx context.Context, items []Item) error + BatchGetItems(ctx context.Context, itemIds []string) ([]Item, error) + DeleteItem(ctx context.Context, itemId string) error + GetItem(ctx context.Context, itemId string) (Item, error) + ModifyItem(ctx context.Context, itemId string, patch ItemPatch) error + GetItems(ctx context.Context, cursor string, n int, beginTime *time.Time) (string, []Item, error) + GetItemFeedback(ctx context.Context, itemId string, feedbackTypes ...string) ([]Feedback, error) + BatchInsertUsers(ctx context.Context, users []User) error + DeleteUser(ctx context.Context, userId string) error + GetUser(ctx context.Context, userId string) (User, error) + ModifyUser(ctx context.Context, userId string, patch UserPatch) error + GetUsers(ctx context.Context, cursor string, n int) (string, []User, error) + GetUserFeedback(ctx context.Context, userId string, endTime *time.Time, feedbackTypes ...string) ([]Feedback, error) + GetUserItemFeedback(ctx context.Context, userId, itemId string, feedbackTypes ...string) ([]Feedback, error) + DeleteUserItemFeedback(ctx context.Context, userId, itemId string, feedbackTypes ...string) (int, error) + BatchInsertFeedback(ctx context.Context, feedback []Feedback, insertUser, insertItem, overwrite bool) error + GetFeedback(ctx context.Context, cursor string, n int, beginTime, endTime *time.Time, feedbackTypes ...string) (string, []Feedback, error) + GetUserStream(ctx context.Context, batchSize int) (chan []User, chan error) + GetItemStream(ctx context.Context, batchSize int, timeLimit *time.Time) (chan []Item, chan error) + GetFeedbackStream(ctx context.Context, batchSize int, beginTime, endTime *time.Time, feedbackTypes ...string) (chan []Feedback, chan error) } // Open a connection to a database. @@ -161,7 +164,10 @@ func Open(path, tablePrefix string) (Database, error) { database := new(SQLDatabase) database.driver = MySQL database.TablePrefix = storage.TablePrefix(tablePrefix) - if database.client, err = sql.Open("mysql", name); err != nil { + if database.client, err = otelsql.Open("mysql", name, + otelsql.WithAttributes(semconv.DBSystemMySQL), + otelsql.WithSpanOptions(otelsql.SpanOptions{DisableErrSkip: true}), + ); err != nil { return nil, errors.Trace(err) } database.gormDB, err = gorm.Open(mysql.New(mysql.Config{Conn: database.client}), storage.NewGORMConfig(tablePrefix)) @@ -173,7 +179,10 @@ func Open(path, tablePrefix string) (Database, error) { database := new(SQLDatabase) database.driver = Postgres database.TablePrefix = storage.TablePrefix(tablePrefix) - if database.client, err = sql.Open("postgres", path); err != nil { + if database.client, err = otelsql.Open("postgres", path, + otelsql.WithAttributes(semconv.DBSystemPostgreSQL), + otelsql.WithSpanOptions(otelsql.SpanOptions{DisableErrSkip: true}), + ); err != nil { return nil, errors.Trace(err) } database.gormDB, err = gorm.Open(postgres.New(postgres.Config{Conn: database.client}), storage.NewGORMConfig(tablePrefix)) @@ -196,7 +205,10 @@ func Open(path, tablePrefix string) (Database, error) { database := new(SQLDatabase) database.driver = ClickHouse database.TablePrefix = storage.TablePrefix(tablePrefix) - if database.client, err = sql.Open("chhttp", uri); err != nil { + if database.client, err = otelsql.Open("chhttp", uri, + otelsql.WithAttributes(semconv.DBSystemKey.String("clickhouse")), + otelsql.WithSpanOptions(otelsql.SpanOptions{DisableErrSkip: true}), + ); err != nil { return nil, errors.Trace(err) } database.gormDB, err = gorm.Open(clickhouse.New(clickhouse.Config{Conn: database.client}), storage.NewGORMConfig(tablePrefix)) @@ -207,7 +219,10 @@ func Open(path, tablePrefix string) (Database, error) { } else if strings.HasPrefix(path, storage.MongoPrefix) || strings.HasPrefix(path, storage.MongoSrvPrefix) { // connect to database database := new(MongoDB) - if database.client, err = mongo.Connect(context.Background(), options.Client().ApplyURI(path)); err != nil { + opts := options.Client() + opts.Monitor = otelmongo.NewMonitor() + opts.ApplyURI(path) + if database.client, err = mongo.Connect(context.Background(), opts); err != nil { return nil, errors.Trace(err) } // parse DSN and extract database name @@ -231,7 +246,10 @@ func Open(path, tablePrefix string) (Database, error) { database := new(SQLDatabase) database.driver = SQLite database.TablePrefix = storage.TablePrefix(tablePrefix) - if database.client, err = sql.Open("sqlite", name); err != nil { + if database.client, err = otelsql.Open("sqlite", name, + otelsql.WithAttributes(semconv.DBSystemSqlite), + otelsql.WithSpanOptions(otelsql.SpanOptions{DisableErrSkip: true}), + ); err != nil { return nil, errors.Trace(err) } gormConfig := storage.NewGORMConfig(tablePrefix) @@ -260,7 +278,10 @@ func Open(path, tablePrefix string) (Database, error) { database := new(SQLDatabase) database.driver = Oracle database.TablePrefix = storage.TablePrefix(tablePrefix) - if database.client, err = sql.Open("oracle", path); err != nil { + if database.client, err = otelsql.Open("oracle", path, + otelsql.WithAttributes(semconv.DBSystemOracle), + otelsql.WithSpanOptions(otelsql.SpanOptions{DisableErrSkip: true}), + ); err != nil { return nil, errors.Trace(err) } database.gormDB, err = gorm.Open(oracle.New(oracle.Config{Conn: database.client}), storage.NewGORMConfig(tablePrefix)) diff --git a/storage/data/database_test.go b/storage/data/database_test.go index 5e458066f..437ab102b 100644 --- a/storage/data/database_test.go +++ b/storage/data/database_test.go @@ -15,6 +15,7 @@ package data import ( + "context" "fmt" "github.com/juju/errors" "github.com/samber/lo" @@ -32,13 +33,14 @@ var ( duplicateFeedbackType = "duplicateFeedbackType" ) -func getUsers(t *testing.T, db Database, batchSize int) []User { +func GetUsers(t *testing.T, db Database, batchSize int) []User { + ctx := context.Background() users := make([]User, 0) var err error var data []User cursor := "" for { - cursor, data, err = db.GetUsers(cursor, batchSize) + cursor, data, err = db.GetUsers(ctx, cursor, batchSize) assert.NoError(t, err) users = append(users, data...) if cursor == "" { @@ -55,8 +57,9 @@ func getUsers(t *testing.T, db Database, batchSize int) []User { } func getUsersStream(t *testing.T, db Database, batchSize int) []User { + ctx := context.Background() var users []User - userChan, errChan := db.GetUserStream(batchSize) + userChan, errChan := db.GetUserStream(ctx, batchSize) for batchUsers := range userChan { users = append(users, batchUsers...) } @@ -64,13 +67,14 @@ func getUsersStream(t *testing.T, db Database, batchSize int) []User { return users } -func getItems(t *testing.T, db Database, batchSize int) []Item { +func GetItems(t *testing.T, db Database, batchSize int) []Item { + ctx := context.Background() items := make([]Item, 0) var err error var data []Item cursor := "" for { - cursor, data, err = db.GetItems(cursor, batchSize, nil) + cursor, data, err = db.GetItems(ctx, cursor, batchSize, nil) assert.NoError(t, err) items = append(items, data...) if cursor == "" { @@ -87,8 +91,9 @@ func getItems(t *testing.T, db Database, batchSize int) []Item { } func getItemStream(t *testing.T, db Database, batchSize int) []Item { + ctx := context.Background() var items []Item - itemChan, errChan := db.GetItemStream(batchSize, nil) + itemChan, errChan := db.GetItemStream(ctx, batchSize, nil) for batchUsers := range itemChan { items = append(items, batchUsers...) } @@ -96,13 +101,14 @@ func getItemStream(t *testing.T, db Database, batchSize int) []Item { return items } -func getFeedback(t *testing.T, db Database, batchSize int, beginTime, endTime *time.Time, feedbackTypes ...string) []Feedback { +func GetFeedback(t *testing.T, db Database, batchSize int, beginTime, endTime *time.Time, feedbackTypes ...string) []Feedback { feedback := make([]Feedback, 0) + ctx := context.Background() var err error var data []Feedback cursor := "" for { - cursor, data, err = db.GetFeedback(cursor, batchSize, beginTime, endTime, feedbackTypes...) + cursor, data, err = db.GetFeedback(ctx, cursor, batchSize, beginTime, endTime, feedbackTypes...) assert.NoError(t, err) feedback = append(feedback, data...) if cursor == "" { @@ -119,8 +125,9 @@ func getFeedback(t *testing.T, db Database, batchSize int, beginTime, endTime *t } func getFeedbackStream(t *testing.T, db Database, batchSize int, beginTime, endTime *time.Time, feedbackTypes ...string) []Feedback { + ctx := context.Background() var feedbacks []Feedback - feedbackChan, errChan := db.GetFeedbackStream(batchSize, beginTime, endTime, feedbackTypes...) + feedbackChan, errChan := db.GetFeedbackStream(ctx, batchSize, beginTime, endTime, feedbackTypes...) for batchFeedback := range feedbackChan { feedbacks = append(feedbacks, batchFeedback...) } @@ -130,6 +137,7 @@ func getFeedbackStream(t *testing.T, db Database, batchSize int, beginTime, endT func testUsers(t *testing.T, db Database) { // Insert users + ctx := context.Background() var insertedUsers []User for i := 9; i >= 0; i-- { insertedUsers = append(insertedUsers, User{ @@ -138,10 +146,10 @@ func testUsers(t *testing.T, db Database) { Comment: fmt.Sprintf("comment %d", i), }) } - err := db.BatchInsertUsers(insertedUsers) + err := db.BatchInsertUsers(ctx, insertedUsers) assert.NoError(t, err) // Get users - users := getUsers(t, db, 3) + users := GetUsers(t, db, 3) assert.Equal(t, 10, len(users)) for i, user := range users { assert.Equal(t, strconv.Itoa(i), user.UserId) @@ -152,52 +160,53 @@ func testUsers(t *testing.T, db Database) { usersFromStream := getUsersStream(t, db, 3) assert.ElementsMatch(t, insertedUsers, usersFromStream) // Get this user - user, err := db.GetUser("0") + user, err := db.GetUser(ctx, "0") assert.NoError(t, err) assert.Equal(t, "0", user.UserId) // Delete this user - err = db.DeleteUser("0") + err = db.DeleteUser(ctx, "0") assert.NoError(t, err) - _, err = db.GetUser("0") + _, err = db.GetUser(ctx, "0") assert.True(t, errors.Is(err, errors.NotFound), err) // test override - err = db.BatchInsertUsers([]User{{UserId: "1", Comment: "override"}}) + err = db.BatchInsertUsers(ctx, []User{{UserId: "1", Comment: "override"}}) assert.NoError(t, err) err = db.Optimize() assert.NoError(t, err) - user, err = db.GetUser("1") + user, err = db.GetUser(ctx, "1") assert.NoError(t, err) assert.Equal(t, "override", user.Comment) // test modify - err = db.ModifyUser("1", UserPatch{Comment: proto.String("modify")}) + err = db.ModifyUser(ctx, "1", UserPatch{Comment: proto.String("modify")}) assert.NoError(t, err) - err = db.ModifyUser("1", UserPatch{Labels: []string{"a", "b", "c"}}) + err = db.ModifyUser(ctx, "1", UserPatch{Labels: []string{"a", "b", "c"}}) assert.NoError(t, err) - err = db.ModifyUser("1", UserPatch{Subscribe: []string{"d", "e", "f"}}) + err = db.ModifyUser(ctx, "1", UserPatch{Subscribe: []string{"d", "e", "f"}}) assert.NoError(t, err) err = db.Optimize() assert.NoError(t, err) - user, err = db.GetUser("1") + user, err = db.GetUser(ctx, "1") assert.NoError(t, err) assert.Equal(t, "modify", user.Comment) assert.Equal(t, []string{"a", "b", "c"}, user.Labels) assert.Equal(t, []string{"d", "e", "f"}, user.Subscribe) // test insert empty - err = db.BatchInsertUsers(nil) + err = db.BatchInsertUsers(ctx, nil) assert.NoError(t, err) // insert duplicate users - err = db.BatchInsertUsers([]User{{UserId: "1"}, {UserId: "1"}}) + err = db.BatchInsertUsers(ctx, []User{{UserId: "1"}, {UserId: "1"}}) assert.NoError(t, err) } func testFeedback(t *testing.T, db Database) { // users that already exists - err := db.BatchInsertUsers([]User{{"0", []string{"a"}, []string{"x"}, "comment"}}) + ctx := context.Background() + err := db.BatchInsertUsers(ctx, []User{{"0", []string{"a"}, []string{"x"}, "comment"}}) assert.NoError(t, err) // items that already exists - err = db.BatchInsertItems([]Item{{ItemId: "0", Labels: []string{"b"}, Timestamp: time.Date(1996, 4, 8, 10, 0, 0, 0, time.UTC)}}) + err = db.BatchInsertItems(ctx, []Item{{ItemId: "0", Labels: []string{"b"}, Timestamp: time.Date(1996, 4, 8, 10, 0, 0, 0, time.UTC)}}) assert.NoError(t, err) // insert feedbacks timestamp := time.Date(1996, 3, 15, 0, 0, 0, 0, time.UTC) @@ -208,12 +217,12 @@ func testFeedback(t *testing.T, db Database) { {FeedbackKey{positiveFeedbackType, "3", "2"}, timestamp, "comment"}, {FeedbackKey{positiveFeedbackType, "4", "0"}, timestamp, "comment"}, } - err = db.BatchInsertFeedback(feedback, true, true, true) + err = db.BatchInsertFeedback(ctx, feedback, true, true, true) assert.NoError(t, err) // other type - err = db.BatchInsertFeedback([]Feedback{{FeedbackKey: FeedbackKey{negativeFeedbackType, "0", "2"}}}, true, true, true) + err = db.BatchInsertFeedback(ctx, []Feedback{{FeedbackKey: FeedbackKey{negativeFeedbackType, "0", "2"}}}, true, true, true) assert.NoError(t, err) - err = db.BatchInsertFeedback([]Feedback{{FeedbackKey: FeedbackKey{negativeFeedbackType, "2", "4"}}}, true, true, true) + err = db.BatchInsertFeedback(ctx, []Feedback{{FeedbackKey: FeedbackKey{negativeFeedbackType, "2", "4"}}}, true, true, true) assert.NoError(t, err) // future feedback futureFeedback := []Feedback{ @@ -223,14 +232,14 @@ func testFeedback(t *testing.T, db Database) { {FeedbackKey{duplicateFeedbackType, "3", "6"}, time.Now().Add(time.Hour), "comment"}, {FeedbackKey{duplicateFeedbackType, "4", "8"}, time.Now().Add(time.Hour), "comment"}, } - err = db.BatchInsertFeedback(futureFeedback, true, true, true) + err = db.BatchInsertFeedback(ctx, futureFeedback, true, true, true) assert.NoError(t, err) // Get feedback - ret := getFeedback(t, db, 3, nil, lo.ToPtr(time.Now()), positiveFeedbackType) + ret := GetFeedback(t, db, 3, nil, lo.ToPtr(time.Now()), positiveFeedbackType) assert.Equal(t, feedback, ret) - ret = getFeedback(t, db, 2, nil, lo.ToPtr(time.Now())) + ret = GetFeedback(t, db, 2, nil, lo.ToPtr(time.Now())) assert.Equal(t, len(feedback)+2, len(ret)) - ret = getFeedback(t, db, 2, lo.ToPtr(timestamp.Add(time.Second)), lo.ToPtr(time.Now())) + ret = GetFeedback(t, db, 2, lo.ToPtr(timestamp.Add(time.Second)), lo.ToPtr(time.Now())) assert.Empty(t, ret) // Get feedback stream feedbackFromStream := getFeedbackStream(t, db, 3, nil, lo.ToPtr(time.Now()), positiveFeedbackType) @@ -242,7 +251,7 @@ func testFeedback(t *testing.T, db Database) { // Get items err = db.Optimize() assert.NoError(t, err) - items := getItems(t, db, 3) + items := GetItems(t, db, 3) assert.Equal(t, 5, len(items)) for i, item := range items { assert.Equal(t, strconv.Itoa(i*2), item.ItemId) @@ -258,7 +267,7 @@ func testFeedback(t *testing.T, db Database) { } } // Get users - users := getUsers(t, db, 2) + users := GetUsers(t, db, 2) assert.Equal(t, 5, len(users)) for i, user := range users { assert.Equal(t, strconv.Itoa(i), user.UserId) @@ -269,88 +278,88 @@ func testFeedback(t *testing.T, db Database) { } } // check users that already exists - user, err := db.GetUser("0") + user, err := db.GetUser(ctx, "0") assert.NoError(t, err) assert.Equal(t, User{"0", []string{"a"}, []string{"x"}, "comment"}, user) // check items that already exists - item, err := db.GetItem("0") + item, err := db.GetItem(ctx, "0") assert.NoError(t, err) assert.Equal(t, Item{ItemId: "0", Labels: []string{"b"}, Timestamp: time.Date(1996, 4, 8, 10, 0, 0, 0, time.UTC)}, item) // Get typed feedback by user - ret, err = db.GetUserFeedback("2", lo.ToPtr(time.Now()), positiveFeedbackType) + ret, err = db.GetUserFeedback(ctx, "2", lo.ToPtr(time.Now()), positiveFeedbackType) assert.NoError(t, err) assert.Equal(t, 1, len(ret)) assert.Equal(t, "2", ret[0].UserId) assert.Equal(t, "4", ret[0].ItemId) // Get all feedback by user - ret, err = db.GetUserFeedback("2", lo.ToPtr(time.Now())) + ret, err = db.GetUserFeedback(ctx, "2", lo.ToPtr(time.Now())) assert.NoError(t, err) assert.Equal(t, 2, len(ret)) // Get typed feedback by item - ret, err = db.GetItemFeedback("4", positiveFeedbackType) + ret, err = db.GetItemFeedback(ctx, "4", positiveFeedbackType) assert.NoError(t, err) assert.Equal(t, 1, len(ret)) assert.Equal(t, "2", ret[0].UserId) assert.Equal(t, "4", ret[0].ItemId) // Get all feedback by item - ret, err = db.GetItemFeedback("4") + ret, err = db.GetItemFeedback(ctx, "4") assert.NoError(t, err) assert.Equal(t, 2, len(ret)) // test override - err = db.BatchInsertFeedback([]Feedback{{ + err = db.BatchInsertFeedback(ctx, []Feedback{{ FeedbackKey: FeedbackKey{positiveFeedbackType, "0", "8"}, Comment: "override", }}, true, true, true) assert.NoError(t, err) err = db.Optimize() assert.NoError(t, err) - ret, err = db.GetUserFeedback("0", lo.ToPtr(time.Now()), positiveFeedbackType) + ret, err = db.GetUserFeedback(ctx, "0", lo.ToPtr(time.Now()), positiveFeedbackType) assert.NoError(t, err) assert.Equal(t, 1, len(ret)) assert.Equal(t, "override", ret[0].Comment) // test not overwrite - err = db.BatchInsertFeedback([]Feedback{{ + err = db.BatchInsertFeedback(ctx, []Feedback{{ FeedbackKey: FeedbackKey{positiveFeedbackType, "0", "8"}, Comment: "not_override", }}, true, true, false) assert.NoError(t, err) err = db.Optimize() assert.NoError(t, err) - ret, err = db.GetUserFeedback("0", lo.ToPtr(time.Now()), positiveFeedbackType) + ret, err = db.GetUserFeedback(ctx, "0", lo.ToPtr(time.Now()), positiveFeedbackType) assert.NoError(t, err) assert.Equal(t, 1, len(ret)) assert.Equal(t, "override", ret[0].Comment) // insert no feedback - err = db.BatchInsertFeedback(nil, true, true, true) + err = db.BatchInsertFeedback(ctx, nil, true, true, true) assert.NoError(t, err) // not insert users or items - err = db.BatchInsertFeedback([]Feedback{ + err = db.BatchInsertFeedback(ctx, []Feedback{ {FeedbackKey: FeedbackKey{"a", "100", "200"}}, {FeedbackKey: FeedbackKey{"a", "0", "200"}}, {FeedbackKey: FeedbackKey{"a", "100", "8"}}, }, false, false, false) assert.NoError(t, err) - result, err := db.GetUserItemFeedback("100", "200") + result, err := db.GetUserItemFeedback(ctx, "100", "200") assert.NoError(t, err) assert.Empty(t, result) - result, err = db.GetUserItemFeedback("0", "200") + result, err = db.GetUserItemFeedback(ctx, "0", "200") assert.NoError(t, err) assert.Empty(t, result) - result, err = db.GetUserItemFeedback("100", "8") + result, err = db.GetUserItemFeedback(ctx, "100", "8") assert.NoError(t, err) assert.Empty(t, result) // insert valid feedback and invalid feedback at the same time - err = db.BatchInsertFeedback([]Feedback{ + err = db.BatchInsertFeedback(ctx, []Feedback{ {FeedbackKey: FeedbackKey{"a", "0", "8"}}, {FeedbackKey: FeedbackKey{"a", "100", "200"}}, }, false, false, false) assert.NoError(t, err) // insert duplicate feedback - err = db.BatchInsertFeedback([]Feedback{ + err = db.BatchInsertFeedback(ctx, []Feedback{ {FeedbackKey: FeedbackKey{"a", "0", "0"}}, {FeedbackKey: FeedbackKey{"a", "0", "0"}}, }, true, true, true) @@ -358,6 +367,7 @@ func testFeedback(t *testing.T, db Database) { } func testItems(t *testing.T, db Database) { + ctx := context.Background() // Items items := []Item{ { @@ -400,36 +410,36 @@ func testItems(t *testing.T, db Database) { }, } // Insert item - err := db.BatchInsertItems(items) + err := db.BatchInsertItems(ctx, items) assert.NoError(t, err) // Get items - totalItems := getItems(t, db, 3) + totalItems := GetItems(t, db, 3) assert.Equal(t, items, totalItems) // Get item stream itemsFromStream := getItemStream(t, db, 3) assert.ElementsMatch(t, items, itemsFromStream) // Get item for _, item := range items { - ret, err := db.GetItem(item.ItemId) + ret, err := db.GetItem(ctx, item.ItemId) assert.NoError(t, err) assert.Equal(t, item, ret) } // batch get items - batchItem, err := db.BatchGetItems([]string{"2", "6"}) + batchItem, err := db.BatchGetItems(ctx, []string{"2", "6"}) assert.NoError(t, err) assert.Equal(t, []Item{items[1], items[3]}, batchItem) // Delete item - err = db.DeleteItem("0") + err = db.DeleteItem(ctx, "0") assert.NoError(t, err) - _, err = db.GetItem("0") + _, err = db.GetItem(ctx, "0") assert.True(t, errors.Is(err, errors.NotFound), err) // test override - err = db.BatchInsertItems([]Item{{ItemId: "4", IsHidden: false, Categories: []string{"b"}, Labels: []string{"o"}, Comment: "override"}}) + err = db.BatchInsertItems(ctx, []Item{{ItemId: "4", IsHidden: false, Categories: []string{"b"}, Labels: []string{"o"}, Comment: "override"}}) assert.NoError(t, err) err = db.Optimize() assert.NoError(t, err) - item, err := db.GetItem("4") + item, err := db.GetItem(ctx, "4") assert.NoError(t, err) assert.False(t, item.IsHidden) assert.Equal(t, []string{"b"}, item.Categories) @@ -438,19 +448,19 @@ func testItems(t *testing.T, db Database) { // test modify timestamp := time.Date(2000, 1, 1, 1, 1, 1, 0, time.UTC) - err = db.ModifyItem("2", ItemPatch{IsHidden: proto.Bool(true)}) + err = db.ModifyItem(ctx, "2", ItemPatch{IsHidden: proto.Bool(true)}) assert.NoError(t, err) - err = db.ModifyItem("2", ItemPatch{Categories: []string{"a"}}) + err = db.ModifyItem(ctx, "2", ItemPatch{Categories: []string{"a"}}) assert.NoError(t, err) - err = db.ModifyItem("2", ItemPatch{Comment: proto.String("modify")}) + err = db.ModifyItem(ctx, "2", ItemPatch{Comment: proto.String("modify")}) assert.NoError(t, err) - err = db.ModifyItem("2", ItemPatch{Labels: []string{"a", "b", "c"}}) + err = db.ModifyItem(ctx, "2", ItemPatch{Labels: []string{"a", "b", "c"}}) assert.NoError(t, err) - err = db.ModifyItem("2", ItemPatch{Timestamp: ×tamp}) + err = db.ModifyItem(ctx, "2", ItemPatch{Timestamp: ×tamp}) assert.NoError(t, err) err = db.Optimize() assert.NoError(t, err) - item, err = db.GetItem("2") + item, err = db.GetItem(ctx, "2") assert.NoError(t, err) assert.True(t, item.IsHidden) assert.Equal(t, []string{"a"}, item.Categories) @@ -459,19 +469,20 @@ func testItems(t *testing.T, db Database) { assert.Equal(t, timestamp, item.Timestamp) // test insert empty - err = db.BatchInsertItems(nil) + err = db.BatchInsertItems(ctx, nil) assert.NoError(t, err) // test get empty - items, err = db.BatchGetItems(nil) + items, err = db.BatchGetItems(ctx, nil) assert.NoError(t, err) assert.Empty(t, items) // test insert duplicate items - err = db.BatchInsertItems([]Item{{ItemId: "1"}, {ItemId: "1"}}) + err = db.BatchInsertItems(ctx, []Item{{ItemId: "1"}, {ItemId: "1"}}) assert.NoError(t, err) } func testDeleteUser(t *testing.T, db Database) { + ctx := context.Background() // Insert ret feedback := []Feedback{ {FeedbackKey{positiveFeedbackType, "a", "0"}, time.Date(1996, 3, 15, 0, 0, 0, 0, time.UTC), "comment"}, @@ -480,22 +491,23 @@ func testDeleteUser(t *testing.T, db Database) { {FeedbackKey{positiveFeedbackType, "a", "6"}, time.Date(1996, 3, 15, 0, 0, 0, 0, time.UTC), "comment"}, {FeedbackKey{positiveFeedbackType, "a", "8"}, time.Date(1996, 3, 15, 0, 0, 0, 0, time.UTC), "comment"}, } - err := db.BatchInsertFeedback(feedback, true, true, true) + err := db.BatchInsertFeedback(ctx, feedback, true, true, true) assert.NoError(t, err) // Delete user - err = db.DeleteUser("a") + err = db.DeleteUser(ctx, "a") assert.NoError(t, err) - _, err = db.GetUser("a") + _, err = db.GetUser(ctx, "a") assert.NotNil(t, err, "failed to delete user") - ret, err := db.GetUserFeedback("a", lo.ToPtr(time.Now()), positiveFeedbackType) + ret, err := db.GetUserFeedback(ctx, "a", lo.ToPtr(time.Now()), positiveFeedbackType) assert.NoError(t, err) assert.Equal(t, 0, len(ret)) - _, ret, err = db.GetFeedback("", 100, nil, lo.ToPtr(time.Now()), positiveFeedbackType) + _, ret, err = db.GetFeedback(ctx, "", 100, nil, lo.ToPtr(time.Now()), positiveFeedbackType) assert.NoError(t, err) assert.Empty(t, ret) } func testDeleteItem(t *testing.T, db Database) { + ctx := context.Background() // Insert ret feedbacks := []Feedback{ {FeedbackKey{positiveFeedbackType, "0", "b"}, time.Date(1996, 3, 15, 0, 0, 0, 0, time.UTC), "comment"}, @@ -504,22 +516,23 @@ func testDeleteItem(t *testing.T, db Database) { {FeedbackKey{positiveFeedbackType, "3", "b"}, time.Date(1996, 3, 15, 0, 0, 0, 0, time.UTC), "comment"}, {FeedbackKey{positiveFeedbackType, "4", "b"}, time.Date(1996, 3, 15, 0, 0, 0, 0, time.UTC), "comment"}, } - err := db.BatchInsertFeedback(feedbacks, true, true, true) + err := db.BatchInsertFeedback(ctx, feedbacks, true, true, true) assert.NoError(t, err) // Delete item - err = db.DeleteItem("b") + err = db.DeleteItem(ctx, "b") assert.NoError(t, err) - _, err = db.GetItem("b") + _, err = db.GetItem(ctx, "b") assert.Error(t, err, "failed to delete item") - ret, err := db.GetItemFeedback("b", positiveFeedbackType) + ret, err := db.GetItemFeedback(ctx, "b", positiveFeedbackType) assert.NoError(t, err) assert.Equal(t, 0, len(ret)) - _, ret, err = db.GetFeedback("", 100, nil, lo.ToPtr(time.Now()), positiveFeedbackType) + _, ret, err = db.GetFeedback(ctx, "", 100, nil, lo.ToPtr(time.Now()), positiveFeedbackType) assert.NoError(t, err) assert.Empty(t, ret) } func testDeleteFeedback(t *testing.T, db Database) { + ctx := context.Background() feedbacks := []Feedback{ {FeedbackKey{"type1", "2", "3"}, time.Date(1996, 3, 15, 0, 0, 0, 0, time.UTC), "comment"}, {FeedbackKey{"type2", "2", "3"}, time.Date(1996, 3, 15, 0, 0, 0, 0, time.UTC), "comment"}, @@ -527,39 +540,40 @@ func testDeleteFeedback(t *testing.T, db Database) { {FeedbackKey{"type1", "2", "4"}, time.Date(1996, 3, 15, 0, 0, 0, 0, time.UTC), "comment"}, {FeedbackKey{"type1", "1", "3"}, time.Date(1996, 3, 15, 0, 0, 0, 0, time.UTC), "comment"}, } - err := db.BatchInsertFeedback(feedbacks, true, true, true) + err := db.BatchInsertFeedback(ctx, feedbacks, true, true, true) assert.NoError(t, err) // get user-item feedback - ret, err := db.GetUserItemFeedback("2", "3") + ret, err := db.GetUserItemFeedback(ctx, "2", "3") assert.NoError(t, err) assert.ElementsMatch(t, []Feedback{feedbacks[0], feedbacks[1], feedbacks[2]}, ret) feedbackType2 := "type2" - ret, err = db.GetUserItemFeedback("2", "3", feedbackType2) + ret, err = db.GetUserItemFeedback(ctx, "2", "3", feedbackType2) assert.NoError(t, err) assert.Equal(t, []Feedback{feedbacks[1]}, ret) // delete user-item feedback - deleteCount, err := db.DeleteUserItemFeedback("2", "3") + deleteCount, err := db.DeleteUserItemFeedback(ctx, "2", "3") assert.NoError(t, err) if !isClickHouse(db) { // RowAffected isn't supported by ClickHouse, assert.Equal(t, 3, deleteCount) } - ret, err = db.GetUserItemFeedback("2", "3") + ret, err = db.GetUserItemFeedback(ctx, "2", "3") assert.NoError(t, err) assert.Empty(t, ret) feedbackType1 := "type1" - deleteCount, err = db.DeleteUserItemFeedback("1", "3", feedbackType1) + deleteCount, err = db.DeleteUserItemFeedback(ctx, "1", "3", feedbackType1) assert.NoError(t, err) if !isClickHouse(db) { // RowAffected isn't supported by ClickHouse, assert.Equal(t, 1, deleteCount) } - ret, err = db.GetUserItemFeedback("1", "3", feedbackType2) + ret, err = db.GetUserItemFeedback(ctx, "1", "3", feedbackType2) assert.NoError(t, err) assert.Empty(t, ret) } func testTimeLimit(t *testing.T, db Database) { + ctx := context.Background() // insert items items := []Item{ { @@ -593,10 +607,10 @@ func testTimeLimit(t *testing.T, db Database) { Comment: "comment 8", }, } - err := db.BatchInsertItems(items) + err := db.BatchInsertItems(ctx, items) assert.NoError(t, err) timeLimit := time.Date(1998, 1, 1, 0, 0, 0, 0, time.UTC) - _, ret, err := db.GetItems("", 100, &timeLimit) + _, ret, err := db.GetItems(ctx, "", 100, &timeLimit) assert.NoError(t, err) assert.Equal(t, []Item{items[2], items[3], items[4]}, ret) @@ -608,22 +622,23 @@ func testTimeLimit(t *testing.T, db Database) { {FeedbackKey{"type1", "2", "4"}, time.Date(1999, 3, 15, 0, 0, 0, 0, time.UTC), "comment"}, {FeedbackKey{"type1", "1", "3"}, time.Date(2000, 3, 15, 0, 0, 0, 0, time.UTC), "comment"}, } - err = db.BatchInsertFeedback(feedbacks, true, true, true) + err = db.BatchInsertFeedback(ctx, feedbacks, true, true, true) assert.NoError(t, err) - _, retFeedback, err := db.GetFeedback("", 100, &timeLimit, lo.ToPtr(time.Now())) + _, retFeedback, err := db.GetFeedback(ctx, "", 100, &timeLimit, lo.ToPtr(time.Now())) assert.NoError(t, err) assert.Equal(t, []Feedback{feedbacks[4], feedbacks[3], feedbacks[2]}, retFeedback) typeFilter := "type1" - _, retFeedback, err = db.GetFeedback("", 100, &timeLimit, lo.ToPtr(time.Now()), typeFilter) + _, retFeedback, err = db.GetFeedback(ctx, "", 100, &timeLimit, lo.ToPtr(time.Now()), typeFilter) assert.NoError(t, err) assert.Equal(t, []Feedback{feedbacks[4], feedbacks[3]}, retFeedback) } func testTimeZone(t *testing.T, db Database) { + ctx := context.Background() loc, err := time.LoadLocation("Asia/Tokyo") assert.NoError(t, err) // insert feedbacks - err = db.BatchInsertFeedback([]Feedback{ + err = db.BatchInsertFeedback(ctx, []Feedback{ {FeedbackKey: FeedbackKey{"read", "1", "1"}, Timestamp: time.Now().Add(-time.Second).In(loc)}, {FeedbackKey: FeedbackKey{"read", "1", "2"}, Timestamp: time.Now().Add(-time.Second).In(loc)}, {FeedbackKey: FeedbackKey{"read", "2", "2"}, Timestamp: time.Now().Add(-time.Second).In(loc)}, @@ -633,30 +648,30 @@ func testTimeZone(t *testing.T, db Database) { }, true, true, true) assert.NoError(t, err) // get feedback stream - feedback := getFeedback(t, db, 10, nil, lo.ToPtr(time.Now())) + feedback := GetFeedback(t, db, 10, nil, lo.ToPtr(time.Now())) assert.Equal(t, 3, len(feedback)) // get feedback - _, feedback, err = db.GetFeedback("", 10, nil, lo.ToPtr(time.Now())) + _, feedback, err = db.GetFeedback(ctx, "", 10, nil, lo.ToPtr(time.Now())) assert.NoError(t, err) assert.Equal(t, 3, len(feedback)) // get user feedback - feedback, err = db.GetUserFeedback("1", lo.ToPtr(time.Now())) + feedback, err = db.GetUserFeedback(ctx, "1", lo.ToPtr(time.Now())) assert.NoError(t, err) assert.Equal(t, 2, len(feedback)) // get item feedback - feedback, err = db.GetItemFeedback("2") // no future feedback by default + feedback, err = db.GetItemFeedback(ctx, "2") // no future feedback by default assert.NoError(t, err) assert.Equal(t, 2, len(feedback)) // get user item feedback - feedback, err = db.GetUserItemFeedback("1", "1") // return future feedback by default + feedback, err = db.GetUserItemFeedback(ctx, "1", "1") // return future feedback by default assert.NoError(t, err) assert.Equal(t, 2, len(feedback)) // insert items now := time.Now().In(loc) - err = db.BatchInsertItems([]Item{{ItemId: "100", Timestamp: now}, {ItemId: "200"}}) + err = db.BatchInsertItems(ctx, []Item{{ItemId: "100", Timestamp: now}, {ItemId: "200"}}) assert.NoError(t, err) - err = db.ModifyItem("200", ItemPatch{Timestamp: &now}) + err = db.ModifyItem(ctx, "200", ItemPatch{Timestamp: &now}) assert.NoError(t, err) err = db.Optimize() assert.NoError(t, err) @@ -664,34 +679,34 @@ func testTimeZone(t *testing.T, db Database) { case *SQLDatabase: switch db.(*SQLDatabase).driver { case Postgres: - item, err := db.GetItem("100") + item, err := db.GetItem(ctx, "100") assert.NoError(t, err) assert.Equal(t, now.Round(time.Microsecond).In(time.UTC), item.Timestamp) - item, err = db.GetItem("200") + item, err = db.GetItem(ctx, "200") assert.NoError(t, err) assert.Equal(t, now.Round(time.Microsecond).In(time.UTC), item.Timestamp) case ClickHouse, Oracle: - item, err := db.GetItem("100") + item, err := db.GetItem(ctx, "100") assert.NoError(t, err) assert.Equal(t, now.Truncate(time.Second).In(time.UTC), item.Timestamp) - item, err = db.GetItem("200") + item, err = db.GetItem(ctx, "200") assert.NoError(t, err) assert.Equal(t, now.Truncate(time.Second).In(time.UTC), item.Timestamp) case SQLite: - item, err := db.GetItem("100") + item, err := db.GetItem(ctx, "100") assert.NoError(t, err) assert.Equal(t, now.In(time.UTC), item.Timestamp.In(time.UTC)) - item, err = db.GetItem("200") + item, err = db.GetItem(ctx, "200") assert.NoError(t, err) assert.Equal(t, now.In(time.UTC), item.Timestamp.In(time.UTC)) default: t.Skipf("unknown sql database: %v", database.driver) } case *MongoDB: - item, err := db.GetItem("100") + item, err := db.GetItem(ctx, "100") assert.NoError(t, err) assert.Equal(t, now.Truncate(time.Millisecond).In(time.UTC), item.Timestamp) - item, err = db.GetItem("200") + item, err = db.GetItem(ctx, "200") assert.NoError(t, err) assert.Equal(t, now.Truncate(time.Millisecond).In(time.UTC), item.Timestamp) default: @@ -708,8 +723,9 @@ func isClickHouse(db Database) bool { } func testPurge(t *testing.T, db Database) { + ctx := context.Background() // insert data - err := db.BatchInsertFeedback(lo.Map(lo.Range(100), func(t int, i int) Feedback { + err := db.BatchInsertFeedback(ctx, lo.Map(lo.Range(100), func(t int, i int) Feedback { return Feedback{FeedbackKey: FeedbackKey{ FeedbackType: "click", UserId: strconv.Itoa(t), @@ -717,25 +733,25 @@ func testPurge(t *testing.T, db Database) { }} }), true, true, true) assert.NoError(t, err) - _, users, err := db.GetUsers("", 100) + _, users, err := db.GetUsers(ctx, "", 100) assert.NoError(t, err) assert.Equal(t, 100, len(users)) - _, items, err := db.GetItems("", 100, nil) + _, items, err := db.GetItems(ctx, "", 100, nil) assert.NoError(t, err) assert.Equal(t, 100, len(items)) - _, feedbacks, err := db.GetFeedback("", 100, nil, lo.ToPtr(time.Now())) + _, feedbacks, err := db.GetFeedback(ctx, "", 100, nil, lo.ToPtr(time.Now())) assert.NoError(t, err) assert.Equal(t, 100, len(feedbacks)) // purge data err = db.Purge() assert.NoError(t, err) - _, users, err = db.GetUsers("", 100) + _, users, err = db.GetUsers(ctx, "", 100) assert.NoError(t, err) assert.Empty(t, users) - _, items, err = db.GetItems("", 100, nil) + _, items, err = db.GetItems(ctx, "", 100, nil) assert.NoError(t, err) assert.Empty(t, items) - _, feedbacks, err = db.GetFeedback("", 100, nil, lo.ToPtr(time.Now())) + _, feedbacks, err = db.GetFeedback(ctx, "", 100, nil, lo.ToPtr(time.Now())) assert.NoError(t, err) assert.Empty(t, feedbacks) // purge empty database diff --git a/storage/data/mongodb.go b/storage/data/mongodb.go index 3cdff98f4..fa7a87b4d 100644 --- a/storage/data/mongodb.go +++ b/storage/data/mongodb.go @@ -150,11 +150,10 @@ func (db *MongoDB) Purge() error { } // BatchInsertItems insert items into MongoDB. -func (db *MongoDB) BatchInsertItems(items []Item) error { +func (db *MongoDB) BatchInsertItems(ctx context.Context, items []Item) error { if len(items) == 0 { return nil } - ctx := context.Background() c := db.client.Database(db.dbName).Collection(db.ItemsTable()) var models []mongo.WriteModel for _, item := range items { @@ -167,11 +166,10 @@ func (db *MongoDB) BatchInsertItems(items []Item) error { return errors.Trace(err) } -func (db *MongoDB) BatchGetItems(itemIds []string) ([]Item, error) { +func (db *MongoDB) BatchGetItems(ctx context.Context, itemIds []string) ([]Item, error) { if len(itemIds) == 0 { return nil, nil } - ctx := context.Background() c := db.client.Database(db.dbName).Collection(db.ItemsTable()) r, err := c.Find(ctx, bson.M{"itemid": bson.M{"$in": itemIds}}) if err != nil { @@ -190,7 +188,7 @@ func (db *MongoDB) BatchGetItems(itemIds []string) ([]Item, error) { } // ModifyItem modify an item in MongoDB. -func (db *MongoDB) ModifyItem(itemId string, patch ItemPatch) error { +func (db *MongoDB) ModifyItem(ctx context.Context, itemId string, patch ItemPatch) error { // create update update := bson.M{} if patch.IsHidden != nil { @@ -209,15 +207,13 @@ func (db *MongoDB) ModifyItem(itemId string, patch ItemPatch) error { update["timestamp"] = patch.Timestamp } // execute - ctx := context.Background() c := db.client.Database(db.dbName).Collection(db.ItemsTable()) _, err := c.UpdateOne(ctx, bson.M{"itemid": bson.M{"$eq": itemId}}, bson.M{"$set": update}) return errors.Trace(err) } // DeleteItem deletes a item from MongoDB. -func (db *MongoDB) DeleteItem(itemId string) error { - ctx := context.Background() +func (db *MongoDB) DeleteItem(ctx context.Context, itemId string) error { c := db.client.Database(db.dbName).Collection(db.ItemsTable()) _, err := c.DeleteOne(ctx, bson.M{"itemid": itemId}) if err != nil { @@ -231,8 +227,7 @@ func (db *MongoDB) DeleteItem(itemId string) error { } // GetItem returns a item from MongoDB. -func (db *MongoDB) GetItem(itemId string) (item Item, err error) { - ctx := context.Background() +func (db *MongoDB) GetItem(ctx context.Context, itemId string) (item Item, err error) { c := db.client.Database(db.dbName).Collection(db.ItemsTable()) r := c.FindOne(ctx, bson.M{"itemid": itemId}) if r.Err() == mongo.ErrNoDocuments { @@ -244,8 +239,7 @@ func (db *MongoDB) GetItem(itemId string) (item Item, err error) { } // GetItems returns items from MongoDB. -func (db *MongoDB) GetItems(cursor string, n int, timeLimit *time.Time) (string, []Item, error) { - ctx := context.Background() +func (db *MongoDB) GetItems(ctx context.Context, cursor string, n int, timeLimit *time.Time) (string, []Item, error) { c := db.client.Database(db.dbName).Collection(db.ItemsTable()) opt := options.Find() opt.SetLimit(int64(n)) @@ -276,7 +270,7 @@ func (db *MongoDB) GetItems(cursor string, n int, timeLimit *time.Time) (string, } // GetItemStream read items from MongoDB by stream. -func (db *MongoDB) GetItemStream(batchSize int, timeLimit *time.Time) (chan []Item, chan error) { +func (db *MongoDB) GetItemStream(ctx context.Context, batchSize int, timeLimit *time.Time) (chan []Item, chan error) { itemChan := make(chan []Item, bufSize) errChan := make(chan error, 1) go func() { @@ -319,8 +313,7 @@ func (db *MongoDB) GetItemStream(batchSize int, timeLimit *time.Time) (chan []It } // GetItemFeedback returns feedback of a item from MongoDB. -func (db *MongoDB) GetItemFeedback(itemId string, feedbackTypes ...string) ([]Feedback, error) { - ctx := context.Background() +func (db *MongoDB) GetItemFeedback(ctx context.Context, itemId string, feedbackTypes ...string) ([]Feedback, error) { c := db.client.Database(db.dbName).Collection(db.FeedbackTable()) var r *mongo.Cursor var err error @@ -348,11 +341,10 @@ func (db *MongoDB) GetItemFeedback(itemId string, feedbackTypes ...string) ([]Fe } // BatchInsertUsers inserts a user into MongoDB. -func (db *MongoDB) BatchInsertUsers(users []User) error { +func (db *MongoDB) BatchInsertUsers(ctx context.Context, users []User) error { if len(users) == 0 { return nil } - ctx := context.Background() c := db.client.Database(db.dbName).Collection(db.UsersTable()) var models []mongo.WriteModel for _, user := range users { @@ -366,7 +358,7 @@ func (db *MongoDB) BatchInsertUsers(users []User) error { } // ModifyUser modify a user in MongoDB. -func (db *MongoDB) ModifyUser(userId string, patch UserPatch) error { +func (db *MongoDB) ModifyUser(ctx context.Context, userId string, patch UserPatch) error { // create patch update := bson.M{} if patch.Labels != nil { @@ -379,15 +371,13 @@ func (db *MongoDB) ModifyUser(userId string, patch UserPatch) error { update["subscribe"] = patch.Subscribe } // execute - ctx := context.Background() c := db.client.Database(db.dbName).Collection(db.UsersTable()) _, err := c.UpdateOne(ctx, bson.M{"userid": bson.M{"$eq": userId}}, bson.M{"$set": update}) return errors.Trace(err) } // DeleteUser deletes a user from MongoDB. -func (db *MongoDB) DeleteUser(userId string) error { - ctx := context.Background() +func (db *MongoDB) DeleteUser(ctx context.Context, userId string) error { c := db.client.Database(db.dbName).Collection(db.UsersTable()) _, err := c.DeleteOne(ctx, bson.M{"userid": userId}) if err != nil { @@ -401,8 +391,7 @@ func (db *MongoDB) DeleteUser(userId string) error { } // GetUser returns a user from MongoDB. -func (db *MongoDB) GetUser(userId string) (user User, err error) { - ctx := context.Background() +func (db *MongoDB) GetUser(ctx context.Context, userId string) (user User, err error) { c := db.client.Database(db.dbName).Collection(db.UsersTable()) r := c.FindOne(ctx, bson.M{"userid": userId}) if r.Err() == mongo.ErrNoDocuments { @@ -414,8 +403,7 @@ func (db *MongoDB) GetUser(userId string) (user User, err error) { } // GetUsers returns users from MongoDB. -func (db *MongoDB) GetUsers(cursor string, n int) (string, []User, error) { - ctx := context.Background() +func (db *MongoDB) GetUsers(ctx context.Context, cursor string, n int) (string, []User, error) { c := db.client.Database(db.dbName).Collection(db.UsersTable()) opt := options.Find() opt.SetLimit(int64(n)) @@ -442,7 +430,7 @@ func (db *MongoDB) GetUsers(cursor string, n int) (string, []User, error) { } // GetUserStream reads users from MongoDB by stream. -func (db *MongoDB) GetUserStream(batchSize int) (chan []User, chan error) { +func (db *MongoDB) GetUserStream(ctx context.Context, batchSize int) (chan []User, chan error) { userChan := make(chan []User, bufSize) errChan := make(chan error, 1) go func() { @@ -480,8 +468,7 @@ func (db *MongoDB) GetUserStream(batchSize int) (chan []User, chan error) { } // GetUserFeedback returns feedback of a user from MongoDB. -func (db *MongoDB) GetUserFeedback(userId string, endTime *time.Time, feedbackTypes ...string) ([]Feedback, error) { - ctx := context.Background() +func (db *MongoDB) GetUserFeedback(ctx context.Context, userId string, endTime *time.Time, feedbackTypes ...string) ([]Feedback, error) { c := db.client.Database(db.dbName).Collection(db.FeedbackTable()) var r *mongo.Cursor var err error @@ -511,8 +498,7 @@ func (db *MongoDB) GetUserFeedback(userId string, endTime *time.Time, feedbackTy } // BatchInsertFeedback returns multiple feedback into MongoDB. -func (db *MongoDB) BatchInsertFeedback(feedback []Feedback, insertUser, insertItem, overwrite bool) error { - ctx := context.Background() +func (db *MongoDB) BatchInsertFeedback(ctx context.Context, feedback []Feedback, insertUser, insertItem, overwrite bool) error { // skip empty list if len(feedback) == 0 { return nil @@ -541,7 +527,7 @@ func (db *MongoDB) BatchInsertFeedback(feedback []Feedback, insertUser, insertIt } } else { for _, userId := range userList { - _, err := db.GetUser(userId) + _, err := db.GetUser(ctx, userId) if err != nil { if errors.Is(err, errors.NotFound) { users.Remove(userId) @@ -568,7 +554,7 @@ func (db *MongoDB) BatchInsertFeedback(feedback []Feedback, insertUser, insertIt } } else { for _, itemId := range itemList { - _, err := db.GetItem(itemId) + _, err := db.GetItem(ctx, itemId) if err != nil { if errors.Is(err, errors.NotFound) { items.Remove(itemId) @@ -604,8 +590,7 @@ func (db *MongoDB) BatchInsertFeedback(feedback []Feedback, insertUser, insertIt } // GetFeedback returns multiple feedback from MongoDB. -func (db *MongoDB) GetFeedback(cursor string, n int, beginTime, endTime *time.Time, feedbackTypes ...string) (string, []Feedback, error) { - ctx := context.Background() +func (db *MongoDB) GetFeedback(ctx context.Context, cursor string, n int, beginTime, endTime *time.Time, feedbackTypes ...string) (string, []Feedback, error) { c := db.client.Database(db.dbName).Collection(db.FeedbackTable()) opt := options.Find() opt.SetLimit(int64(n)) @@ -657,7 +642,7 @@ func (db *MongoDB) GetFeedback(cursor string, n int, beginTime, endTime *time.Ti } // GetFeedbackStream reads feedback from MongoDB by stream. -func (db *MongoDB) GetFeedbackStream(batchSize int, beginTime, endTime *time.Time, feedbackTypes ...string) (chan []Feedback, chan error) { +func (db *MongoDB) GetFeedbackStream(ctx context.Context, batchSize int, beginTime, endTime *time.Time, feedbackTypes ...string) (chan []Feedback, chan error) { feedbackChan := make(chan []Feedback, bufSize) errChan := make(chan error, 1) go func() { @@ -709,8 +694,7 @@ func (db *MongoDB) GetFeedbackStream(batchSize int, beginTime, endTime *time.Tim } // GetUserItemFeedback returns a feedback return the user id and item id from MongoDB. -func (db *MongoDB) GetUserItemFeedback(userId, itemId string, feedbackTypes ...string) ([]Feedback, error) { - ctx := context.Background() +func (db *MongoDB) GetUserItemFeedback(ctx context.Context, userId, itemId string, feedbackTypes ...string) ([]Feedback, error) { c := db.client.Database(db.dbName).Collection(db.FeedbackTable()) var filter = bson.M{ "feedbackkey.userid": bson.M{"$eq": userId}, @@ -736,8 +720,7 @@ func (db *MongoDB) GetUserItemFeedback(userId, itemId string, feedbackTypes ...s } // DeleteUserItemFeedback deletes a feedback return the user id and item id from MongoDB. -func (db *MongoDB) DeleteUserItemFeedback(userId, itemId string, feedbackTypes ...string) (int, error) { - ctx := context.Background() +func (db *MongoDB) DeleteUserItemFeedback(ctx context.Context, userId, itemId string, feedbackTypes ...string) (int, error) { c := db.client.Database(db.dbName).Collection(db.FeedbackTable()) var filter = bson.M{ "feedbackkey.userid": bson.M{"$eq": userId}, diff --git a/storage/data/no_database.go b/storage/data/no_database.go index cef9034ec..966210846 100644 --- a/storage/data/no_database.go +++ b/storage/data/no_database.go @@ -15,6 +15,7 @@ package data import ( + "context" "time" ) @@ -41,32 +42,32 @@ func (NoDatabase) Purge() error { } // BatchInsertItems method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) BatchInsertItems(_ []Item) error { +func (NoDatabase) BatchInsertItems(_ context.Context, _ []Item) error { return ErrNoDatabase } // BatchGetItems method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) BatchGetItems(_ []string) ([]Item, error) { +func (NoDatabase) BatchGetItems(_ context.Context, _ []string) ([]Item, error) { return nil, ErrNoDatabase } // DeleteItem method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) DeleteItem(_ string) error { +func (NoDatabase) DeleteItem(_ context.Context, _ string) error { return ErrNoDatabase } // GetItem method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) GetItem(_ string) (Item, error) { +func (NoDatabase) GetItem(_ context.Context, _ string) (Item, error) { return Item{}, ErrNoDatabase } // GetItems method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) GetItems(_ string, _ int, _ *time.Time) (string, []Item, error) { +func (NoDatabase) GetItems(_ context.Context, _ string, _ int, _ *time.Time) (string, []Item, error) { return "", nil, ErrNoDatabase } // GetItemStream method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) GetItemStream(_ int, _ *time.Time) (chan []Item, chan error) { +func (NoDatabase) GetItemStream(_ context.Context, _ int, _ *time.Time) (chan []Item, chan error) { itemChan := make(chan []Item, bufSize) errChan := make(chan error, 1) go func() { @@ -78,32 +79,32 @@ func (NoDatabase) GetItemStream(_ int, _ *time.Time) (chan []Item, chan error) { } // GetItemFeedback method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) GetItemFeedback(_ string, _ ...string) ([]Feedback, error) { +func (NoDatabase) GetItemFeedback(_ context.Context, _ string, _ ...string) ([]Feedback, error) { return nil, ErrNoDatabase } // BatchInsertUsers method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) BatchInsertUsers(_ []User) error { +func (NoDatabase) BatchInsertUsers(_ context.Context, _ []User) error { return ErrNoDatabase } // DeleteUser method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) DeleteUser(_ string) error { +func (NoDatabase) DeleteUser(_ context.Context, _ string) error { return ErrNoDatabase } // GetUser method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) GetUser(_ string) (User, error) { +func (NoDatabase) GetUser(_ context.Context, _ string) (User, error) { return User{}, ErrNoDatabase } // GetUsers method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) GetUsers(_ string, _ int) (string, []User, error) { +func (NoDatabase) GetUsers(_ context.Context, _ string, _ int) (string, []User, error) { return "", nil, ErrNoDatabase } // GetUserStream method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) GetUserStream(_ int) (chan []User, chan error) { +func (NoDatabase) GetUserStream(_ context.Context, _ int) (chan []User, chan error) { userChan := make(chan []User, bufSize) errChan := make(chan error, 1) go func() { @@ -115,32 +116,32 @@ func (NoDatabase) GetUserStream(_ int) (chan []User, chan error) { } // GetUserFeedback method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) GetUserFeedback(_ string, _ *time.Time, _ ...string) ([]Feedback, error) { +func (NoDatabase) GetUserFeedback(_ context.Context, _ string, _ *time.Time, _ ...string) ([]Feedback, error) { return nil, ErrNoDatabase } // GetUserItemFeedback method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) GetUserItemFeedback(_, _ string, _ ...string) ([]Feedback, error) { +func (NoDatabase) GetUserItemFeedback(_ context.Context, _, _ string, _ ...string) ([]Feedback, error) { return nil, ErrNoDatabase } // DeleteUserItemFeedback method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) DeleteUserItemFeedback(_, _ string, _ ...string) (int, error) { +func (NoDatabase) DeleteUserItemFeedback(_ context.Context, _, _ string, _ ...string) (int, error) { return 0, ErrNoDatabase } // BatchInsertFeedback method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) BatchInsertFeedback(_ []Feedback, _, _, _ bool) error { +func (NoDatabase) BatchInsertFeedback(_ context.Context, _ []Feedback, _, _, _ bool) error { return ErrNoDatabase } // GetFeedback method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) GetFeedback(_ string, _ int, _, _ *time.Time, _ ...string) (string, []Feedback, error) { +func (NoDatabase) GetFeedback(_ context.Context, _ string, _ int, _, _ *time.Time, _ ...string) (string, []Feedback, error) { return "", nil, ErrNoDatabase } // GetFeedbackStream method of NoDatabase returns ErrNoDatabase. -func (NoDatabase) GetFeedbackStream(_ int, _, _ *time.Time, _ ...string) (chan []Feedback, chan error) { +func (NoDatabase) GetFeedbackStream(_ context.Context, _ int, _, _ *time.Time, _ ...string) (chan []Feedback, chan error) { feedbackChan := make(chan []Feedback, bufSize) errChan := make(chan error, 1) go func() { @@ -151,10 +152,10 @@ func (NoDatabase) GetFeedbackStream(_ int, _, _ *time.Time, _ ...string) (chan [ return feedbackChan, errChan } -func (d NoDatabase) ModifyItem(_ string, _ ItemPatch) error { +func (d NoDatabase) ModifyItem(_ context.Context, _ string, _ ItemPatch) error { return ErrNoDatabase } -func (d NoDatabase) ModifyUser(_ string, _ UserPatch) error { +func (d NoDatabase) ModifyUser(_ context.Context, _ string, _ UserPatch) error { return ErrNoDatabase } diff --git a/storage/data/no_database_test.go b/storage/data/no_database_test.go index 32880f958..5a17bea11 100644 --- a/storage/data/no_database_test.go +++ b/storage/data/no_database_test.go @@ -15,6 +15,7 @@ package data import ( + "context" "github.com/samber/lo" "github.com/stretchr/testify/assert" "testing" @@ -22,6 +23,7 @@ import ( ) func TestNoDatabase(t *testing.T) { + ctx := context.Background() var database NoDatabase err := database.Close() @@ -33,48 +35,48 @@ func TestNoDatabase(t *testing.T) { err = database.Purge() assert.ErrorIs(t, err, ErrNoDatabase) - err = database.BatchInsertItems(nil) + err = database.BatchInsertItems(ctx, nil) assert.ErrorIs(t, err, ErrNoDatabase) - _, err = database.BatchGetItems(nil) + _, err = database.BatchGetItems(ctx, nil) assert.ErrorIs(t, err, ErrNoDatabase) - err = database.ModifyItem("", ItemPatch{}) + err = database.ModifyItem(ctx, "", ItemPatch{}) assert.ErrorIs(t, err, ErrNoDatabase) - _, err = database.GetItem("") + _, err = database.GetItem(ctx, "") assert.ErrorIs(t, err, ErrNoDatabase) - _, _, err = database.GetItems("", 0, nil) + _, _, err = database.GetItems(ctx, "", 0, nil) assert.ErrorIs(t, err, ErrNoDatabase) - err = database.DeleteItem("") + err = database.DeleteItem(ctx, "") assert.ErrorIs(t, err, ErrNoDatabase) - _, c := database.GetItemStream(0, nil) + _, c := database.GetItemStream(ctx, 0, nil) assert.ErrorIs(t, <-c, ErrNoDatabase) - err = database.BatchInsertUsers(nil) + err = database.BatchInsertUsers(ctx, nil) assert.ErrorIs(t, err, ErrNoDatabase) - _, err = database.GetUser("") + _, err = database.GetUser(ctx, "") assert.ErrorIs(t, err, ErrNoDatabase) - err = database.ModifyUser("", UserPatch{}) + err = database.ModifyUser(ctx, "", UserPatch{}) assert.ErrorIs(t, err, ErrNoDatabase) - _, _, err = database.GetUsers("", 0) + _, _, err = database.GetUsers(ctx, "", 0) assert.ErrorIs(t, err, ErrNoDatabase) - err = database.DeleteUser("") + err = database.DeleteUser(ctx, "") assert.ErrorIs(t, err, ErrNoDatabase) - _, c = database.GetUserStream(0) + _, c = database.GetUserStream(ctx, 0) assert.ErrorIs(t, <-c, ErrNoDatabase) - err = database.BatchInsertFeedback(nil, false, false, false) + err = database.BatchInsertFeedback(ctx, nil, false, false, false) assert.ErrorIs(t, err, ErrNoDatabase) - err = database.BatchInsertFeedback(nil, false, false, false) + err = database.BatchInsertFeedback(ctx, nil, false, false, false) assert.ErrorIs(t, err, ErrNoDatabase) - _, err = database.GetUserFeedback("", lo.ToPtr(time.Now())) + _, err = database.GetUserFeedback(ctx, "", lo.ToPtr(time.Now())) assert.ErrorIs(t, err, ErrNoDatabase) - _, err = database.GetItemFeedback("") + _, err = database.GetItemFeedback(ctx, "") assert.ErrorIs(t, err, ErrNoDatabase) - _, _, err = database.GetFeedback("", 0, nil, lo.ToPtr(time.Now())) + _, _, err = database.GetFeedback(ctx, "", 0, nil, lo.ToPtr(time.Now())) assert.ErrorIs(t, err, ErrNoDatabase) - _, err = database.GetUserItemFeedback("", "") + _, err = database.GetUserItemFeedback(ctx, "", "") assert.ErrorIs(t, err, ErrNoDatabase) - _, err = database.DeleteUserItemFeedback("", "") + _, err = database.DeleteUserItemFeedback(ctx, "", "") assert.ErrorIs(t, err, ErrNoDatabase) - _, c = database.GetFeedbackStream(0, nil, lo.ToPtr(time.Now())) + _, c = database.GetFeedbackStream(ctx, 0, nil, lo.ToPtr(time.Now())) assert.ErrorIs(t, <-c, ErrNoDatabase) } diff --git a/storage/data/redis.go b/storage/data/redis.go index 3dee123cb..52ab0e381 100644 --- a/storage/data/redis.go +++ b/storage/data/redis.go @@ -57,8 +57,7 @@ func (r *Redis) Purge() error { } // insertItem inserts an item into Redis. -func (r *Redis) insertItem(item Item) error { - var ctx = context.Background() +func (r *Redis) insertItem(ctx context.Context, item Item) error { // write item data, err := json.Marshal(item) if err != nil { @@ -71,17 +70,16 @@ func (r *Redis) insertItem(item Item) error { } // BatchInsertItems inserts a batch of items into Redis. -func (r *Redis) BatchInsertItems(items []Item) error { +func (r *Redis) BatchInsertItems(ctx context.Context, items []Item) error { for _, item := range items { - if err := r.insertItem(item); err != nil { + if err := r.insertItem(ctx, item); err != nil { return errors.Trace(err) } } return nil } -func (r *Redis) BatchGetItems(itemIds []string) ([]Item, error) { - ctx := context.Background() +func (r *Redis) BatchGetItems(ctx context.Context, itemIds []string) ([]Item, error) { var ( items []Item cursor uint64 @@ -137,8 +135,7 @@ func (r *Redis) ForFeedback(ctx context.Context, action func(key, thisFeedbackTy } // DeleteItem deletes a item from Redis. -func (r *Redis) DeleteItem(itemId string) error { - var ctx = context.Background() +func (r *Redis) DeleteItem(ctx context.Context, itemId string) error { // remove user if err := r.client.Del(ctx, prefixItem+itemId).Err(); err != nil { return errors.Trace(err) @@ -154,8 +151,7 @@ func (r *Redis) DeleteItem(itemId string) error { } // GetItem get a item from Redis. -func (r *Redis) GetItem(itemId string) (Item, error) { - var ctx = context.Background() +func (r *Redis) GetItem(ctx context.Context, itemId string) (Item, error) { data, err := r.client.Get(ctx, prefixItem+itemId).Result() if err != nil { if err == redis.Nil { @@ -169,8 +165,7 @@ func (r *Redis) GetItem(itemId string) (Item, error) { } // GetItems returns items from Redis. -func (r *Redis) GetItems(cursor string, n int, timeLimit *time.Time) (string, []Item, error) { - var ctx = context.Background() +func (r *Redis) GetItems(ctx context.Context, cursor string, n int, timeLimit *time.Time) (string, []Item, error) { var err error cursorNum := uint64(0) if len(cursor) > 0 { @@ -211,7 +206,7 @@ func (r *Redis) GetItems(cursor string, n int, timeLimit *time.Time) (string, [] } // GetItemStream read items from Redis by stream. -func (r *Redis) GetItemStream(batchSize int, timeLimit *time.Time) (chan []Item, chan error) { +func (r *Redis) GetItemStream(ctx context.Context, batchSize int, timeLimit *time.Time) (chan []Item, chan error) { itemChan := make(chan []Item, bufSize) errChan := make(chan error, 1) go func() { @@ -263,8 +258,7 @@ func (r *Redis) GetItemStream(batchSize int, timeLimit *time.Time) (chan []Item, } // GetItemFeedback returns feedback of an item from Redis. -func (r *Redis) GetItemFeedback(itemId string, feedbackTypes ...string) ([]Feedback, error) { - var ctx = context.Background() +func (r *Redis) GetItemFeedback(ctx context.Context, itemId string, feedbackTypes ...string) ([]Feedback, error) { feedback := make([]Feedback, 0) feedbackTypeSet := strset.New(feedbackTypes...) err := r.ForFeedback(ctx, func(key, thisFeedbackType, _, thisItemId string) error { @@ -283,8 +277,7 @@ func (r *Redis) GetItemFeedback(itemId string, feedbackTypes ...string) ([]Feedb } // insertUser inserts a user into Redis. -func (r *Redis) insertUser(user User) error { - var ctx = context.Background() +func (r *Redis) insertUser(ctx context.Context, user User) error { data, err := json.Marshal(user) if err != nil { return errors.Trace(err) @@ -293,9 +286,9 @@ func (r *Redis) insertUser(user User) error { } // BatchInsertUsers inserts a batch pf user into Redis. -func (r *Redis) BatchInsertUsers(users []User) error { +func (r *Redis) BatchInsertUsers(ctx context.Context, users []User) error { for _, user := range users { - if err := r.insertUser(user); err != nil { + if err := r.insertUser(ctx, user); err != nil { return err } } @@ -303,8 +296,7 @@ func (r *Redis) BatchInsertUsers(users []User) error { } // DeleteUser deletes a user from Redis. -func (r *Redis) DeleteUser(userId string) error { - var ctx = context.Background() +func (r *Redis) DeleteUser(ctx context.Context, userId string) error { // remove user if err := r.client.Del(ctx, prefixUser+userId).Err(); err != nil { return errors.Trace(err) @@ -319,8 +311,7 @@ func (r *Redis) DeleteUser(userId string) error { } // GetUser returns a user from Redis. -func (r *Redis) GetUser(userId string) (User, error) { - var ctx = context.Background() +func (r *Redis) GetUser(ctx context.Context, userId string) (User, error) { val, err := r.client.Get(ctx, prefixUser+userId).Result() if err != nil { if err == redis.Nil { @@ -337,8 +328,7 @@ func (r *Redis) GetUser(userId string) (User, error) { } // GetUsers returns users from Redis. -func (r *Redis) GetUsers(cursor string, n int) (string, []User, error) { - var ctx = context.Background() +func (r *Redis) GetUsers(ctx context.Context, cursor string, n int) (string, []User, error) { var err error cursorNum := uint64(0) if len(cursor) > 0 { @@ -374,7 +364,7 @@ func (r *Redis) GetUsers(cursor string, n int) (string, []User, error) { } // GetUserStream read users from Redis by stream. -func (r *Redis) GetUserStream(batchSize int) (chan []User, chan error) { +func (r *Redis) GetUserStream(ctx context.Context, batchSize int) (chan []User, chan error) { userChan := make(chan []User, bufSize) errChan := make(chan error, 1) go func() { @@ -422,8 +412,7 @@ func (r *Redis) GetUserStream(batchSize int) (chan []User, chan error) { } // GetUserFeedback returns feedback of a user from Redis. -func (r *Redis) GetUserFeedback(userId string, endTime *time.Time, feedbackTypes ...string) ([]Feedback, error) { - var ctx = context.Background() +func (r *Redis) GetUserFeedback(ctx context.Context, userId string, endTime *time.Time, feedbackTypes ...string) ([]Feedback, error) { feedback := make([]Feedback, 0) feedbackTypeSet := strset.New(feedbackTypes...) // get itemId list by userId @@ -469,9 +458,9 @@ func parseFeedbackKey(key string) (feedbackType, userId, itemId string) { // insertFeedback insert a feedback into Redis. // If insertUser set, a new user will be insert to user table. // If insertItem set, a new item will be insert to item table. -func (r *Redis) insertFeedback(feedback Feedback, insertUser, insertItem, overwrite bool) error { +func (r *Redis) insertFeedback(ctx context.Context, feedback Feedback, insertUser, insertItem, overwrite bool) error { // locate user - _, err := r.GetUser(feedback.UserId) + _, err := r.GetUser(ctx, feedback.UserId) if errors.Is(err, errors.NotFound) { if !insertUser { return nil @@ -480,7 +469,7 @@ func (r *Redis) insertFeedback(feedback Feedback, insertUser, insertItem, overwr return err } // locate item - _, err = r.GetItem(feedback.ItemId) + _, err = r.GetItem(ctx, feedback.ItemId) if errors.Is(err, errors.NotFound) { if !insertItem { return nil @@ -488,7 +477,6 @@ func (r *Redis) insertFeedback(feedback Feedback, insertUser, insertItem, overwr } else if err != nil { return err } - var ctx = context.Background() val, err := json.Marshal(feedback) if err != nil { return errors.Trace(err) @@ -540,9 +528,9 @@ func (r *Redis) insertFeedback(feedback Feedback, insertUser, insertItem, overwr // BatchInsertFeedback insert a batch feedback into Redis. // If insertUser set, new users will be insert to user table. // If insertItem set, new items will be insert to item table. -func (r *Redis) BatchInsertFeedback(feedback []Feedback, insertUser, insertItem, overwrite bool) error { +func (r *Redis) BatchInsertFeedback(ctx context.Context, feedback []Feedback, insertUser, insertItem, overwrite bool) error { for _, temp := range feedback { - if err := r.insertFeedback(temp, insertUser, insertItem, overwrite); err != nil { + if err := r.insertFeedback(ctx, temp, insertUser, insertItem, overwrite); err != nil { return errors.Trace(err) } } @@ -550,8 +538,7 @@ func (r *Redis) BatchInsertFeedback(feedback []Feedback, insertUser, insertItem, } // GetFeedback returns feedback from Redis. -func (r *Redis) GetFeedback(_ string, _ int, beginTime, endTime *time.Time, feedbackTypes ...string) (string, []Feedback, error) { - var ctx = context.Background() +func (r *Redis) GetFeedback(ctx context.Context, _ string, _ int, beginTime, endTime *time.Time, feedbackTypes ...string) (string, []Feedback, error) { feedback := make([]Feedback, 0) feedbackTypeSet := strset.New(feedbackTypes...) err := r.ForFeedback(ctx, func(key, thisFeedbackType, thisUserId, thisItemId string) error { @@ -574,7 +561,7 @@ func (r *Redis) GetFeedback(_ string, _ int, beginTime, endTime *time.Time, feed } // GetFeedbackStream reads feedback by stream. -func (r *Redis) GetFeedbackStream(batchSize int, beginTime, endTime *time.Time, feedbackTypes ...string) (chan []Feedback, chan error) { +func (r *Redis) GetFeedbackStream(ctx context.Context, batchSize int, beginTime, endTime *time.Time, feedbackTypes ...string) (chan []Feedback, chan error) { feedbackChan := make(chan []Feedback, bufSize) errChan := make(chan error, 1) go func() { @@ -612,8 +599,7 @@ func (r *Redis) GetFeedbackStream(batchSize int, beginTime, endTime *time.Time, } // GetUserItemFeedback gets a feedback by user id and item id from Redis. -func (r *Redis) GetUserItemFeedback(userId, itemId string, feedbackTypes ...string) ([]Feedback, error) { - var ctx = context.Background() +func (r *Redis) GetUserItemFeedback(ctx context.Context, userId, itemId string, feedbackTypes ...string) ([]Feedback, error) { feedback := make([]Feedback, 0) feedbackTypeSet := strset.New(feedbackTypes...) err := r.ForFeedback(ctx, func(key, thisFeedbackType, thisUserId, thisItemId string) error { @@ -630,8 +616,7 @@ func (r *Redis) GetUserItemFeedback(userId, itemId string, feedbackTypes ...stri } // DeleteUserItemFeedback deletes a feedback by user id and item id from Redis. -func (r *Redis) DeleteUserItemFeedback(userId, itemId string, feedbackTypes ...string) (int, error) { - var ctx = context.Background() +func (r *Redis) DeleteUserItemFeedback(ctx context.Context, userId, itemId string, feedbackTypes ...string) (int, error) { feedbackTypeSet := strset.New(feedbackTypes...) deleteCount := 0 err := r.ForFeedback(ctx, func(key, thisFeedbackType, thisUserId, thisItemId string) error { @@ -645,9 +630,9 @@ func (r *Redis) DeleteUserItemFeedback(userId, itemId string, feedbackTypes ...s } // ModifyItem modify an item in Redis. -func (r *Redis) ModifyItem(itemId string, patch ItemPatch) error { +func (r *Redis) ModifyItem(ctx context.Context, itemId string, patch ItemPatch) error { // read item - item, err := r.GetItem(itemId) + item, err := r.GetItem(ctx, itemId) if err != nil { return err } @@ -668,13 +653,13 @@ func (r *Redis) ModifyItem(itemId string, patch ItemPatch) error { item.Timestamp = *patch.Timestamp } // write back - return r.insertItem(item) + return r.insertItem(ctx, item) } // ModifyUser modify a user in Redis. -func (r *Redis) ModifyUser(userId string, patch UserPatch) error { +func (r *Redis) ModifyUser(ctx context.Context, userId string, patch UserPatch) error { // read user - user, err := r.GetUser(userId) + user, err := r.GetUser(ctx, userId) if err != nil { return err } @@ -689,5 +674,5 @@ func (r *Redis) ModifyUser(userId string, patch UserPatch) error { user.Subscribe = patch.Subscribe } // write back - return r.insertUser(user) + return r.insertUser(ctx, user) } diff --git a/storage/data/sql.go b/storage/data/sql.go index 166543696..9ad10057b 100644 --- a/storage/data/sql.go +++ b/storage/data/sql.go @@ -15,6 +15,7 @@ package data import ( + "context" "database/sql" "fmt" _ "github.com/go-sql-driver/mysql" @@ -313,7 +314,7 @@ func (d *SQLDatabase) Purge() error { } // BatchInsertItems inserts a batch of items into MySQL. -func (d *SQLDatabase) BatchInsertItems(items []Item) error { +func (d *SQLDatabase) BatchInsertItems(ctx context.Context, items []Item) error { if len(items) == 0 { return nil } @@ -341,7 +342,7 @@ func (d *SQLDatabase) BatchInsertItems(items []Item) error { rows = append(rows, row) } } - err := d.gormDB.Clauses(clause.OnConflict{ + err := d.gormDB.WithContext(ctx).Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "item_id"}}, DoUpdates: clause.AssignmentColumns([]string{"is_hidden", "categories", "time_stamp", "labels", "comment"}), }).Create(rows).Error @@ -349,11 +350,13 @@ func (d *SQLDatabase) BatchInsertItems(items []Item) error { } } -func (d *SQLDatabase) BatchGetItems(itemIds []string) ([]Item, error) { +func (d *SQLDatabase) BatchGetItems(ctx context.Context, itemIds []string) ([]Item, error) { if len(itemIds) == 0 { return nil, nil } - result, err := d.gormDB.Table(d.ItemsTable()).Select("item_id, is_hidden, categories, time_stamp, labels, comment").Where("item_id IN ?", itemIds).Rows() + result, err := d.gormDB.WithContext(ctx).Table(d.ItemsTable()). + Select("item_id, is_hidden, categories, time_stamp, labels, comment"). + Where("item_id IN ?", itemIds).Rows() if err != nil { return nil, errors.Trace(err) } @@ -377,21 +380,21 @@ func (d *SQLDatabase) BatchGetItems(itemIds []string) ([]Item, error) { } // DeleteItem deletes a item from MySQL. -func (d *SQLDatabase) DeleteItem(itemId string) error { - if err := d.gormDB.Delete(&SQLItem{ItemId: itemId}).Error; err != nil { +func (d *SQLDatabase) DeleteItem(ctx context.Context, itemId string) error { + if err := d.gormDB.WithContext(ctx).Delete(&SQLItem{ItemId: itemId}).Error; err != nil { return errors.Trace(err) } - if err := d.gormDB.Delete(&Feedback{}, "item_id = ?", itemId).Error; err != nil { + if err := d.gormDB.WithContext(ctx).Delete(&Feedback{}, "item_id = ?", itemId).Error; err != nil { return errors.Trace(err) } return nil } // GetItem get a item from MySQL. -func (d *SQLDatabase) GetItem(itemId string) (Item, error) { +func (d *SQLDatabase) GetItem(ctx context.Context, itemId string) (Item, error) { var result *sql.Rows var err error - result, err = d.gormDB.Table(d.ItemsTable()).Select("item_id, is_hidden, categories, time_stamp, labels, comment").Where("item_id = ?", itemId).Rows() + result, err = d.gormDB.WithContext(ctx).Table(d.ItemsTable()).Select("item_id, is_hidden, categories, time_stamp, labels, comment").Where("item_id = ?", itemId).Rows() if err != nil { return Item{}, errors.Trace(err) } @@ -416,7 +419,7 @@ func (d *SQLDatabase) GetItem(itemId string) (Item, error) { } // ModifyItem modify an item in MySQL. -func (d *SQLDatabase) ModifyItem(itemId string, patch ItemPatch) error { +func (d *SQLDatabase) ModifyItem(ctx context.Context, itemId string, patch ItemPatch) error { // ignore empty patch if patch.IsHidden == nil && patch.Categories == nil && patch.Labels == nil && patch.Comment == nil && patch.Timestamp == nil { log.Logger().Debug("empty item patch") @@ -449,13 +452,13 @@ func (d *SQLDatabase) ModifyItem(itemId string, patch ItemPatch) error { attributes["time_stamp"] = patch.Timestamp } } - err := d.gormDB.Model(&SQLItem{ItemId: itemId}).Updates(attributes).Error + err := d.gormDB.WithContext(ctx).Model(&SQLItem{ItemId: itemId}).Updates(attributes).Error return errors.Trace(err) } // GetItems returns items from MySQL. -func (d *SQLDatabase) GetItems(cursor string, n int, timeLimit *time.Time) (string, []Item, error) { - tx := d.gormDB.Table(d.ItemsTable()).Select("item_id, is_hidden, categories, time_stamp, labels, comment") +func (d *SQLDatabase) GetItems(ctx context.Context, cursor string, n int, timeLimit *time.Time) (string, []Item, error) { + tx := d.gormDB.WithContext(ctx).Table(d.ItemsTable()).Select("item_id, is_hidden, categories, time_stamp, labels, comment") if cursor != "" { tx.Where("item_id >= ?", cursor) } @@ -491,14 +494,14 @@ func (d *SQLDatabase) GetItems(cursor string, n int, timeLimit *time.Time) (stri } // GetItemStream reads items by stream. -func (d *SQLDatabase) GetItemStream(batchSize int, timeLimit *time.Time) (chan []Item, chan error) { +func (d *SQLDatabase) GetItemStream(ctx context.Context, batchSize int, timeLimit *time.Time) (chan []Item, chan error) { itemChan := make(chan []Item, bufSize) errChan := make(chan error, 1) go func() { defer close(itemChan) defer close(errChan) // send query - tx := d.gormDB.Table(d.ItemsTable()).Select("item_id, is_hidden, categories, time_stamp, labels, comment") + tx := d.gormDB.WithContext(ctx).Table(d.ItemsTable()).Select("item_id, is_hidden, categories, time_stamp, labels, comment") if timeLimit != nil { tx.Where("time_stamp >= ?", *timeLimit) } @@ -540,8 +543,8 @@ func (d *SQLDatabase) GetItemStream(batchSize int, timeLimit *time.Time) (chan [ } // GetItemFeedback returns feedback of a item from MySQL. -func (d *SQLDatabase) GetItemFeedback(itemId string, feedbackTypes ...string) ([]Feedback, error) { - tx := d.gormDB.Table(d.FeedbackTable()).Select("user_id, item_id, feedback_type, time_stamp") +func (d *SQLDatabase) GetItemFeedback(ctx context.Context, itemId string, feedbackTypes ...string) ([]Feedback, error) { + tx := d.gormDB.WithContext(ctx).Table(d.FeedbackTable()).Select("user_id, item_id, feedback_type, time_stamp") switch d.driver { case SQLite: tx.Where("time_stamp <= DATETIME() AND item_id = ?", itemId) @@ -570,7 +573,7 @@ func (d *SQLDatabase) GetItemFeedback(itemId string, feedbackTypes ...string) ([ } // BatchInsertUsers inserts users into MySQL. -func (d *SQLDatabase) BatchInsertUsers(users []User) error { +func (d *SQLDatabase) BatchInsertUsers(ctx context.Context, users []User) error { if len(users) == 0 { return nil } @@ -594,7 +597,7 @@ func (d *SQLDatabase) BatchInsertUsers(users []User) error { rows = append(rows, NewSQLUser(user)) } } - err := d.gormDB.Clauses(clause.OnConflict{ + err := d.gormDB.WithContext(ctx).Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "user_id"}}, DoUpdates: clause.AssignmentColumns([]string{"labels", "subscribe", "comment"}), }).Create(rows).Error @@ -603,21 +606,23 @@ func (d *SQLDatabase) BatchInsertUsers(users []User) error { } // DeleteUser deletes a user from MySQL. -func (d *SQLDatabase) DeleteUser(userId string) error { - if err := d.gormDB.Delete(&SQLUser{UserId: userId}).Error; err != nil { +func (d *SQLDatabase) DeleteUser(ctx context.Context, userId string) error { + if err := d.gormDB.WithContext(ctx).Delete(&SQLUser{UserId: userId}).Error; err != nil { return errors.Trace(err) } - if err := d.gormDB.Delete(&Feedback{}, "user_id = ?", userId).Error; err != nil { + if err := d.gormDB.WithContext(ctx).Delete(&Feedback{}, "user_id = ?", userId).Error; err != nil { return errors.Trace(err) } return nil } // GetUser returns a user from MySQL. -func (d *SQLDatabase) GetUser(userId string) (User, error) { +func (d *SQLDatabase) GetUser(ctx context.Context, userId string) (User, error) { var result *sql.Rows var err error - result, err = d.gormDB.Table(d.UsersTable()).Select("user_id, labels, subscribe, comment").Where("user_id = ?", userId).Rows() + result, err = d.gormDB.WithContext(ctx).Table(d.UsersTable()). + Select("user_id, labels, subscribe, comment"). + Where("user_id = ?", userId).Rows() if err != nil { return User{}, errors.Trace(err) } @@ -641,7 +646,7 @@ func (d *SQLDatabase) GetUser(userId string) (User, error) { } // ModifyUser modify a user in MySQL. -func (d *SQLDatabase) ModifyUser(userId string, patch UserPatch) error { +func (d *SQLDatabase) ModifyUser(ctx context.Context, userId string, patch UserPatch) error { // ignore empty patch if patch.Labels == nil && patch.Subscribe == nil && patch.Comment == nil { log.Logger().Debug("empty user patch") @@ -659,13 +664,13 @@ func (d *SQLDatabase) ModifyUser(userId string, patch UserPatch) error { text, _ := json.Marshal(patch.Subscribe) attributes["subscribe"] = string(text) } - err := d.gormDB.Model(&SQLUser{UserId: userId}).Updates(attributes).Error + err := d.gormDB.WithContext(ctx).Model(&SQLUser{UserId: userId}).Updates(attributes).Error return errors.Trace(err) } // GetUsers returns users from MySQL. -func (d *SQLDatabase) GetUsers(cursor string, n int) (string, []User, error) { - tx := d.gormDB.Table(d.UsersTable()).Select("user_id, labels, subscribe, comment") +func (d *SQLDatabase) GetUsers(ctx context.Context, cursor string, n int) (string, []User, error) { + tx := d.gormDB.WithContext(ctx).Table(d.UsersTable()).Select("user_id, labels, subscribe, comment") if cursor != "" { tx.Where("user_id >= ?", cursor) } @@ -698,14 +703,14 @@ func (d *SQLDatabase) GetUsers(cursor string, n int) (string, []User, error) { } // GetUserStream read users by stream. -func (d *SQLDatabase) GetUserStream(batchSize int) (chan []User, chan error) { +func (d *SQLDatabase) GetUserStream(ctx context.Context, batchSize int) (chan []User, chan error) { userChan := make(chan []User, bufSize) errChan := make(chan error, 1) go func() { defer close(userChan) defer close(errChan) // send query - result, err := d.gormDB.Table(d.UsersTable()).Select("user_id, labels, subscribe, comment").Rows() + result, err := d.gormDB.WithContext(ctx).Table(d.UsersTable()).Select("user_id, labels, subscribe, comment").Rows() if err != nil { errChan <- errors.Trace(err) return @@ -744,8 +749,10 @@ func (d *SQLDatabase) GetUserStream(batchSize int) (chan []User, chan error) { } // GetUserFeedback returns feedback of a user from MySQL. -func (d *SQLDatabase) GetUserFeedback(userId string, endTime *time.Time, feedbackTypes ...string) ([]Feedback, error) { - tx := d.gormDB.Table(d.FeedbackTable()).Select("feedback_type, user_id, item_id, time_stamp, comment").Where("user_id = ?", userId) +func (d *SQLDatabase) GetUserFeedback(ctx context.Context, userId string, endTime *time.Time, feedbackTypes ...string) ([]Feedback, error) { + tx := d.gormDB.WithContext(ctx).Table(d.FeedbackTable()). + Select("feedback_type, user_id, item_id, time_stamp, comment"). + Where("user_id = ?", userId) if endTime != nil { tx.Where("time_stamp <= ?", d.convertTimeZone(endTime)) } @@ -773,7 +780,8 @@ func (d *SQLDatabase) GetUserFeedback(userId string, endTime *time.Time, feedbac // BatchInsertFeedback insert a batch feedback into MySQL. // If insertUser set, new users will be inserted to user table. // If insertItem set, new items will be inserted to item table. -func (d *SQLDatabase) BatchInsertFeedback(feedback []Feedback, insertUser, insertItem, overwrite bool) error { +func (d *SQLDatabase) BatchInsertFeedback(ctx context.Context, feedback []Feedback, insertUser, insertItem, overwrite bool) error { + tx := d.gormDB.WithContext(ctx) // skip empty list if len(feedback) == 0 { return nil @@ -789,7 +797,7 @@ func (d *SQLDatabase) BatchInsertFeedback(feedback []Feedback, insertUser, inser if insertUser { userList := users.List() if d.driver == ClickHouse { - err := d.gormDB.Create(lo.Map(userList, func(userId string, _ int) ClickhouseUser { + err := tx.Create(lo.Map(userList, func(userId string, _ int) ClickhouseUser { return ClickhouseUser{ SQLUser: SQLUser{ UserId: userId, @@ -802,7 +810,7 @@ func (d *SQLDatabase) BatchInsertFeedback(feedback []Feedback, insertUser, inser return errors.Trace(err) } } else { - err := d.gormDB.Clauses(clause.OnConflict{ + err := tx.Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "user_id"}}, DoNothing: true, }).Create(lo.Map(userList, func(userId string, _ int) SQLUser { @@ -818,7 +826,7 @@ func (d *SQLDatabase) BatchInsertFeedback(feedback []Feedback, insertUser, inser } } else { for _, user := range users.List() { - rs, err := d.gormDB.Table(d.UsersTable()).Select("user_id").Where("user_id = ?", user).Rows() + rs, err := tx.Table(d.UsersTable()).Select("user_id").Where("user_id = ?", user).Rows() if err != nil { return errors.Trace(err) } else if !rs.Next() { @@ -833,7 +841,7 @@ func (d *SQLDatabase) BatchInsertFeedback(feedback []Feedback, insertUser, inser if insertItem { itemList := items.List() if d.driver == ClickHouse { - err := d.gormDB.Create(lo.Map(itemList, func(itemId string, _ int) ClickHouseItem { + err := tx.Create(lo.Map(itemList, func(itemId string, _ int) ClickHouseItem { return ClickHouseItem{ SQLItem: SQLItem{ ItemId: itemId, @@ -846,7 +854,7 @@ func (d *SQLDatabase) BatchInsertFeedback(feedback []Feedback, insertUser, inser return errors.Trace(err) } } else { - err := d.gormDB.Clauses(clause.OnConflict{ + err := tx.Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "item_id"}}, DoNothing: true, }).Create(lo.Map(itemList, func(itemId string, _ int) SQLItem { @@ -862,7 +870,7 @@ func (d *SQLDatabase) BatchInsertFeedback(feedback []Feedback, insertUser, inser } } else { for _, item := range items.List() { - rs, err := d.gormDB.Table(d.ItemsTable()).Select("item_id").Where("item_id = ?", item).Rows() + rs, err := tx.Table(d.ItemsTable()).Select("item_id").Where("item_id = ?", item).Rows() if err != nil { return errors.Trace(err) } else if !rs.Next() { @@ -892,7 +900,7 @@ func (d *SQLDatabase) BatchInsertFeedback(feedback []Feedback, insertUser, inser if len(rows) == 0 { return nil } - err := d.gormDB.Create(rows).Error + err := tx.Create(rows).Error return errors.Trace(err) } else { rows := make([]Feedback, 0, len(feedback)) @@ -911,7 +919,7 @@ func (d *SQLDatabase) BatchInsertFeedback(feedback []Feedback, insertUser, inser if len(rows) == 0 { return nil } - err := d.gormDB.Clauses(clause.OnConflict{ + err := tx.Clauses(clause.OnConflict{ Columns: []clause.Column{{Name: "feedback_type"}, {Name: "user_id"}, {Name: "item_id"}}, DoNothing: !overwrite, DoUpdates: lo.If(overwrite, clause.AssignmentColumns([]string{"time_stamp", "comment"})).Else(nil), @@ -921,8 +929,8 @@ func (d *SQLDatabase) BatchInsertFeedback(feedback []Feedback, insertUser, inser } // GetFeedback returns feedback from MySQL. -func (d *SQLDatabase) GetFeedback(cursor string, n int, beginTime, endTime *time.Time, feedbackTypes ...string) (string, []Feedback, error) { - tx := d.gormDB.Table(d.FeedbackTable()).Select("feedback_type, user_id, item_id, time_stamp, comment") +func (d *SQLDatabase) GetFeedback(ctx context.Context, cursor string, n int, beginTime, endTime *time.Time, feedbackTypes ...string) (string, []Feedback, error) { + tx := d.gormDB.WithContext(ctx).Table(d.FeedbackTable()).Select("feedback_type, user_id, item_id, time_stamp, comment") if cursor != "" { var cursorKey FeedbackKey if err := json.Unmarshal([]byte(cursor), &cursorKey); err != nil { @@ -974,14 +982,14 @@ func (d *SQLDatabase) GetFeedback(cursor string, n int, beginTime, endTime *time } // GetFeedbackStream reads feedback by stream. -func (d *SQLDatabase) GetFeedbackStream(batchSize int, beginTime, endTime *time.Time, feedbackTypes ...string) (chan []Feedback, chan error) { +func (d *SQLDatabase) GetFeedbackStream(ctx context.Context, batchSize int, beginTime, endTime *time.Time, feedbackTypes ...string) (chan []Feedback, chan error) { feedbackChan := make(chan []Feedback, bufSize) errChan := make(chan error, 1) go func() { defer close(feedbackChan) defer close(errChan) // send query - tx := d.gormDB.Table(d.FeedbackTable()).Select("feedback_type, user_id, item_id, time_stamp, comment") + tx := d.gormDB.WithContext(ctx).Table(d.FeedbackTable()).Select("feedback_type, user_id, item_id, time_stamp, comment") if len(feedbackTypes) > 0 { tx.Where("feedback_type IN ?", feedbackTypes) } @@ -1022,8 +1030,10 @@ func (d *SQLDatabase) GetFeedbackStream(batchSize int, beginTime, endTime *time. } // GetUserItemFeedback gets a feedback by user id and item id from MySQL. -func (d *SQLDatabase) GetUserItemFeedback(userId, itemId string, feedbackTypes ...string) ([]Feedback, error) { - tx := d.gormDB.Table(d.FeedbackTable()).Select("feedback_type, user_id, item_id, time_stamp, comment").Where("user_id = ? AND item_id = ?", userId, itemId) +func (d *SQLDatabase) GetUserItemFeedback(ctx context.Context, userId, itemId string, feedbackTypes ...string) ([]Feedback, error) { + tx := d.gormDB.WithContext(ctx).Table(d.FeedbackTable()). + Select("feedback_type, user_id, item_id, time_stamp, comment"). + Where("user_id = ? AND item_id = ?", userId, itemId) if len(feedbackTypes) > 0 { tx.Where("feedback_type IN ?", feedbackTypes) } @@ -1046,8 +1056,8 @@ func (d *SQLDatabase) GetUserItemFeedback(userId, itemId string, feedbackTypes . } // DeleteUserItemFeedback deletes a feedback by user id and item id from MySQL. -func (d *SQLDatabase) DeleteUserItemFeedback(userId, itemId string, feedbackTypes ...string) (int, error) { - tx := d.gormDB.Where("user_id = ? AND item_id = ?", userId, itemId) +func (d *SQLDatabase) DeleteUserItemFeedback(ctx context.Context, userId, itemId string, feedbackTypes ...string) (int, error) { + tx := d.gormDB.WithContext(ctx).Where("user_id = ? AND item_id = ?", userId, itemId) if len(feedbackTypes) > 0 { tx.Where("feedback_type IN ?", feedbackTypes) } diff --git a/worker/worker.go b/worker/worker.go index 145644f5f..c78708017 100644 --- a/worker/worker.go +++ b/worker/worker.go @@ -428,6 +428,7 @@ func (w *Worker) estimateRecommendComplexity(numUsers, numItems int) int { // 7. Rank items in results by click-through-rate. // 8. Refresh cache. func (w *Worker) Recommend(users []data.User) { + ctx := context.Background() startRecommendTime := time.Now() log.Logger().Info("ranking recommendation", zap.Int("n_working_users", len(users)), @@ -435,7 +436,7 @@ func (w *Worker) Recommend(users []data.User) { zap.Int("cache_size", w.Config.Recommend.CacheSize)) // pull items from database - itemCache, itemCategories, err := w.pullItems() + itemCache, itemCategories, err := w.pullItems(ctx) if err != nil { log.Logger().Error("failed to pull items", zap.Error(err)) return @@ -506,7 +507,7 @@ func (w *Worker) Recommend(users []data.User) { w.rankingIndex, recall = builder.Build(w.Config.Recommend.Collaborative.IndexRecall, w.Config.Recommend.Collaborative.IndexFitEpoch, false, recommendTask) CollaborativeFilteringIndexRecall.Set(float64(recall)) - if err = w.CacheClient.Set(cache.String(cache.Key(cache.GlobalMeta, cache.MatchingIndexRecall), encoding.FormatFloat32(recall))); err != nil { + if err = w.CacheClient.Set(ctx, cache.String(cache.Key(cache.GlobalMeta, cache.MatchingIndexRecall), encoding.FormatFloat32(recall))); err != nil { log.Logger().Error("failed to write meta", zap.Error(err)) } log.Logger().Info("complete building ranking index", @@ -536,7 +537,7 @@ func (w *Worker) Recommend(users []data.User) { user := users[jobId] userId := user.UserId // skip inactive users before max recommend period - if !w.checkRecommendCacheTimeout(userId, itemCategories) { + if !w.checkRecommendCacheTimeout(ctx, userId, itemCategories) { return nil } updateUserCount.Add(1) @@ -553,7 +554,7 @@ func (w *Worker) Recommend(users []data.User) { // load positive items var positiveItems []string if w.Config.Recommend.Offline.EnableItemBasedRecommend { - positiveItems, err = userFeedbackCache.GetUserFeedback(userId) + positiveItems, err = userFeedbackCache.GetUserFeedback(ctx, userId) if err != nil { log.Logger().Error("failed to pull user feedback", zap.String("user_id", userId), zap.Error(err)) @@ -606,7 +607,7 @@ func (w *Worker) Recommend(users []data.User) { scores := make(map[string]float64) for _, itemId := range positiveItems { // load similar items - similarItems, err := w.CacheClient.GetSorted(cache.Key(cache.ItemNeighbors, itemId, category), 0, w.Config.Recommend.CacheSize) + similarItems, err := w.CacheClient.GetSorted(ctx, cache.Key(cache.ItemNeighbors, itemId, category), 0, w.Config.Recommend.CacheSize) if err != nil { log.Logger().Error("failed to load similar items", zap.Error(err)) return errors.Trace(err) @@ -618,7 +619,7 @@ func (w *Worker) Recommend(users []data.User) { } } // load item neighbors digest - digest, err := w.CacheClient.Get(cache.Key(cache.ItemNeighborsDigest, itemId)).String() + digest, err := w.CacheClient.Get(ctx, cache.Key(cache.ItemNeighborsDigest, itemId)).String() if err != nil { if !errors.Is(err, errors.NotFound) { log.Logger().Error("failed to load item neighbors digest", zap.Error(err)) @@ -644,14 +645,14 @@ func (w *Worker) Recommend(users []data.User) { localStartTime := time.Now() scores := make(map[string]float64) // load similar users - similarUsers, err := w.CacheClient.GetSorted(cache.Key(cache.UserNeighbors, userId), 0, w.Config.Recommend.CacheSize) + similarUsers, err := w.CacheClient.GetSorted(ctx, cache.Key(cache.UserNeighbors, userId), 0, w.Config.Recommend.CacheSize) if err != nil { log.Logger().Error("failed to load similar users", zap.Error(err)) return errors.Trace(err) } for _, user := range similarUsers { // load historical feedback - similarUserPositiveItems, err := userFeedbackCache.GetUserFeedback(user.Id) + similarUserPositiveItems, err := userFeedbackCache.GetUserFeedback(ctx, user.Id) if err != nil { log.Logger().Error("failed to pull user feedback", zap.String("user_id", userId), zap.Error(err)) @@ -665,7 +666,7 @@ func (w *Worker) Recommend(users []data.User) { } } // load user neighbors digest - digest, err := w.CacheClient.Get(cache.Key(cache.UserNeighborsDigest, user.Id)).String() + digest, err := w.CacheClient.Get(ctx, cache.Key(cache.UserNeighborsDigest, user.Id)).String() if err != nil { if !errors.Is(err, errors.NotFound) { log.Logger().Error("failed to load user neighbors digest", zap.Error(err)) @@ -697,7 +698,7 @@ func (w *Worker) Recommend(users []data.User) { if w.Config.Recommend.Offline.EnableLatestRecommend { localStartTime := time.Now() for _, category := range append([]string{""}, itemCategories...) { - latestItems, err := w.CacheClient.GetSorted(cache.Key(cache.LatestItems, category), 0, w.Config.Recommend.CacheSize) + latestItems, err := w.CacheClient.GetSorted(ctx, cache.Key(cache.LatestItems, category), 0, w.Config.Recommend.CacheSize) if err != nil { log.Logger().Error("failed to load latest items", zap.Error(err)) return errors.Trace(err) @@ -717,7 +718,7 @@ func (w *Worker) Recommend(users []data.User) { if w.Config.Recommend.Offline.EnablePopularRecommend { localStartTime := time.Now() for _, category := range append([]string{""}, itemCategories...) { - popularItems, err := w.CacheClient.GetSorted(cache.Key(cache.PopularItems, category), 0, w.Config.Recommend.CacheSize) + popularItems, err := w.CacheClient.GetSorted(ctx, cache.Key(cache.PopularItems, category), 0, w.Config.Recommend.CacheSize) if err != nil { log.Logger().Error("failed to load popular items", zap.Error(err)) return errors.Trace(err) @@ -775,12 +776,13 @@ func (w *Worker) Recommend(users []data.User) { return errors.Trace(err) } - if err = w.CacheClient.SetSorted(cache.Key(cache.OfflineRecommend, userId, category), results[category]); err != nil { + if err = w.CacheClient.SetSorted(ctx, cache.Key(cache.OfflineRecommend, userId, category), results[category]); err != nil { log.Logger().Error("failed to cache recommendation", zap.Error(err)) return errors.Trace(err) } } if err = w.CacheClient.Set( + ctx, cache.Time(cache.Key(cache.LastUpdateUserRecommendTime, userId), time.Now()), cache.String(cache.Key(cache.OfflineRecommendDigest, userId), w.Config.OfflineRecommendDigest( config.WithCollaborative(collaborativeUsed), @@ -792,7 +794,7 @@ func (w *Worker) Recommend(users []data.User) { } // refresh cache - err = w.refreshCache(userId) + err = w.refreshCache(ctx, userId) if err != nil { log.Logger().Error("failed to refresh cache", zap.Error(err)) return errors.Trace(err) @@ -822,6 +824,7 @@ func (w *Worker) Recommend(users []data.User) { } func (w *Worker) collaborativeRecommendBruteForce(userId string, itemCategories []string, excludeSet *strset.Set, itemCache *ItemCache) (map[string][]string, time.Duration, error) { + ctx := context.Background() userIndex := w.RankingModel.GetUserIndex().ToNumber(userId) itemIds := w.RankingModel.GetItemIndex().GetNames() localStartTime := time.Now() @@ -844,7 +847,7 @@ func (w *Worker) collaborativeRecommendBruteForce(userId string, itemCategories for category, recItemsFilter := range recItemsFilters { recommendItems, recommendScores := recItemsFilter.PopAll() recommend[category] = recommendItems - if err := w.CacheClient.SetSorted(cache.Key(cache.CollaborativeRecommend, userId, category), cache.CreateScoredItems(recommendItems, recommendScores)); err != nil { + if err := w.CacheClient.SetSorted(ctx, cache.Key(cache.CollaborativeRecommend, userId, category), cache.CreateScoredItems(recommendItems, recommendScores)); err != nil { log.Logger().Error("failed to cache collaborative filtering recommendation result", zap.String("user_id", userId), zap.Error(err)) return nil, 0, errors.Trace(err) } @@ -853,6 +856,7 @@ func (w *Worker) collaborativeRecommendBruteForce(userId string, itemCategories } func (w *Worker) collaborativeRecommendHNSW(rankingIndex *search.HNSW, userId string, itemCategories []string, excludeSet *strset.Set, itemCache *ItemCache) (map[string][]string, time.Duration, error) { + ctx := context.Background() userIndex := w.RankingModel.GetUserIndex().ToNumber(userId) localStartTime := time.Now() values, scores := rankingIndex.MultiSearch(search.NewDenseVector(w.RankingModel.GetUserFactor(userIndex), nil, false), @@ -870,7 +874,7 @@ func (w *Worker) collaborativeRecommendHNSW(rankingIndex *search.HNSW, userId st } } recommend[category] = recommendItems - if err := w.CacheClient.SetSorted(cache.Key(cache.CollaborativeRecommend, userId, category), + if err := w.CacheClient.SetSorted(ctx, cache.Key(cache.CollaborativeRecommend, userId, category), cache.CreateScoredItems(recommendItems, recommendScores)); err != nil { log.Logger().Error("failed to cache collaborative filtering recommendation result", zap.String("user_id", userId), zap.Error(err)) return nil, 0, errors.Trace(err) @@ -966,6 +970,7 @@ func mergeAndShuffle(candidates [][]string) []cache.Scored { func (w *Worker) exploreRecommend(exploitRecommend []cache.Scored, excludeSet *strset.Set, category string) ([]cache.Scored, error) { var localExcludeSet *strset.Set + ctx := context.Background() if w.Config.Recommend.Replacement.EnableReplacement { localExcludeSet = strset.New() } else { @@ -981,12 +986,12 @@ func (w *Worker) exploreRecommend(exploitRecommend []cache.Scored, excludeSet *s exploreLatestThreshold += threshold } // load popular items - popularItems, err := w.CacheClient.GetSorted(cache.Key(cache.PopularItems, category), 0, w.Config.Recommend.CacheSize) + popularItems, err := w.CacheClient.GetSorted(ctx, cache.Key(cache.PopularItems, category), 0, w.Config.Recommend.CacheSize) if err != nil { return nil, errors.Trace(err) } // load the latest items - latestItems, err := w.CacheClient.GetSorted(cache.Key(cache.LatestItems, category), 0, w.Config.Recommend.CacheSize) + latestItems, err := w.CacheClient.GetSorted(ctx, cache.Key(cache.LatestItems, category), 0, w.Config.Recommend.CacheSize) if err != nil { return nil, errors.Trace(err) } @@ -1028,7 +1033,7 @@ func (w *Worker) exploreRecommend(exploitRecommend []cache.Scored, excludeSet *s // 1. if cache is empty, stale. // 2. if active time > recommend time, stale. // 3. if recommend time + timeout < now, stale. -func (w *Worker) checkRecommendCacheTimeout(userId string, categories []string) bool { +func (w *Worker) checkRecommendCacheTimeout(ctx context.Context, userId string, categories []string) bool { var ( activeTime time.Time recommendTime time.Time @@ -1037,7 +1042,7 @@ func (w *Worker) checkRecommendCacheTimeout(userId string, categories []string) ) // check cache for _, category := range append([]string{""}, categories...) { - items, err := w.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, userId, category), 0, -1) + items, err := w.CacheClient.GetSorted(ctx, cache.Key(cache.OfflineRecommend, userId, category), 0, -1) if err != nil { log.Logger().Error("failed to load offline recommendation", zap.String("user_id", userId), zap.Error(err)) return true @@ -1046,7 +1051,7 @@ func (w *Worker) checkRecommendCacheTimeout(userId string, categories []string) } } // read digest - cacheDigest, err = w.CacheClient.Get(cache.Key(cache.OfflineRecommendDigest, userId)).String() + cacheDigest, err = w.CacheClient.Get(ctx, cache.Key(cache.OfflineRecommendDigest, userId)).String() if err != nil { if !errors.Is(err, errors.NotFound) { log.Logger().Error("failed to load offline recommendation digest", zap.String("user_id", userId), zap.Error(err)) @@ -1057,7 +1062,7 @@ func (w *Worker) checkRecommendCacheTimeout(userId string, categories []string) return true } // read active time - activeTime, err = w.CacheClient.Get(cache.Key(cache.LastModifyUserTime, userId)).Time() + activeTime, err = w.CacheClient.Get(ctx, cache.Key(cache.LastModifyUserTime, userId)).Time() if err != nil { if !errors.Is(err, errors.NotFound) { log.Logger().Error("failed to read last modify user time", zap.Error(err)) @@ -1065,7 +1070,7 @@ func (w *Worker) checkRecommendCacheTimeout(userId string, categories []string) return true } // read recommend time - recommendTime, err = w.CacheClient.Get(cache.Key(cache.LastUpdateUserRecommendTime, userId)).Time() + recommendTime, err = w.CacheClient.Get(ctx, cache.Key(cache.LastUpdateUserRecommendTime, userId)).Time() if err != nil { if !errors.Is(err, errors.NotFound) { log.Logger().Error("failed to read last update user recommend time", zap.Error(err)) @@ -1086,7 +1091,8 @@ func (w *Worker) checkRecommendCacheTimeout(userId string, categories []string) func (w *Worker) loadUserHistoricalItems(database data.Database, userId string) ([]string, []data.Feedback, error) { items := make([]string, 0) - feedbacks, err := database.GetUserFeedback(userId, w.Config.Now()) + ctx := context.Background() + feedbacks, err := database.GetUserFeedback(ctx, userId, w.Config.Now()) if err != nil { return nil, nil, err } @@ -1096,10 +1102,10 @@ func (w *Worker) loadUserHistoricalItems(database data.Database, userId string) return items, feedbacks, nil } -func (w *Worker) refreshCache(userId string) error { +func (w *Worker) refreshCache(ctx context.Context, userId string) error { var timeLimit *time.Time // read recommend time - recommendTime, err := w.CacheClient.Get(cache.Key(cache.LastUpdateUserRecommendTime, userId)).Time() + recommendTime, err := w.CacheClient.Get(ctx, cache.Key(cache.LastUpdateUserRecommendTime, userId)).Time() if err == nil { timeLimit = &recommendTime } else if !errors.Is(err, errors.NotFound) { @@ -1107,12 +1113,12 @@ func (w *Worker) refreshCache(userId string) error { } // reload cache if w.Config.Recommend.Replacement.EnableReplacement { - err = w.CacheClient.SetSorted(cache.IgnoreItems, nil) + err = w.CacheClient.SetSorted(ctx, cache.IgnoreItems, nil) if err != nil { return errors.Trace(err) } } else { - feedback, err := w.DataClient.GetUserFeedback(userId, nil) + feedback, err := w.DataClient.GetUserFeedback(ctx, userId, nil) if err != nil { return errors.Trace(err) } @@ -1122,11 +1128,11 @@ func (w *Worker) refreshCache(userId string) error { items = append(items, cache.Scored{Id: v.ItemId, Score: float64(v.Timestamp.Unix())}) } } - err = w.CacheClient.AddSorted(cache.Sorted(cache.Key(cache.IgnoreItems, userId), items)) + err = w.CacheClient.AddSorted(ctx, cache.Sorted(cache.Key(cache.IgnoreItems, userId), items)) if err != nil { return errors.Trace(err) } - err = w.CacheClient.RemSortedByScore(cache.Key(cache.IgnoreItems, userId), math.Inf(-1), float64(timeLimit.Unix())-1) + err = w.CacheClient.RemSortedByScore(ctx, cache.Key(cache.IgnoreItems, userId), math.Inf(-1), float64(timeLimit.Unix())-1) if err != nil { return errors.Trace(err) } @@ -1134,11 +1140,11 @@ func (w *Worker) refreshCache(userId string) error { return nil } -func (w *Worker) pullItems() (*ItemCache, []string, error) { +func (w *Worker) pullItems(ctx context.Context) (*ItemCache, []string, error) { // pull items from database itemCache := NewItemCache() itemCategories := strset.New() - itemChan, errChan := w.DataClient.GetItemStream(batchSize, nil) + itemChan, errChan := w.DataClient.GetItemStream(ctx, batchSize, nil) for batchItems := range itemChan { for _, item := range batchItems { itemCache.Set(item.ItemId, item) @@ -1152,6 +1158,7 @@ func (w *Worker) pullItems() (*ItemCache, []string, error) { } func (w *Worker) pullUsers(peers []string, me string) ([]data.User, error) { + ctx := context.Background() // locate me if !funk.ContainsString(peers, me) { return nil, errors.New("current node isn't in worker nodes") @@ -1163,7 +1170,7 @@ func (w *Worker) pullUsers(peers []string, me string) ([]data.User, error) { } // pull users from database var users []data.User - userChan, errChan := w.DataClient.GetUserStream(batchSize) + userChan, errChan := w.DataClient.GetUserStream(ctx, batchSize) for batchUsers := range userChan { for _, user := range batchUsers { p, err := c.Get(user.UserId) @@ -1334,12 +1341,12 @@ func NewFeedbackCache(worker *Worker, feedbackTypes ...string) *FeedbackCache { } // GetUserFeedback gets user feedback from cache or database. -func (c *FeedbackCache) GetUserFeedback(userId string) ([]string, error) { +func (c *FeedbackCache) GetUserFeedback(ctx context.Context, userId string) ([]string, error) { if tmp, ok := c.Cache.Get(userId); ok { return tmp.([]string), nil } else { items := make([]string, 0) - feedbacks, err := c.Client.GetUserFeedback(userId, c.Config.Now(), c.Types...) + feedbacks, err := c.Client.GetUserFeedback(ctx, userId, c.Config.Now(), c.Types...) if err != nil { return nil, err } diff --git a/worker/worker_test.go b/worker/worker_test.go index 647f50bb8..f8b4e1b17 100644 --- a/worker/worker_test.go +++ b/worker/worker_test.go @@ -46,8 +46,10 @@ func TestPullUsers(t *testing.T) { // create mock worker w := newMockWorker(t) defer w.Close(t) + + ctx := context.Background() // create user index - err := w.DataClient.BatchInsertUsers([]data.User{ + err := w.DataClient.BatchInsertUsers(ctx, []data.User{ {UserId: "1"}, {UserId: "2"}, {UserId: "3"}, @@ -73,29 +75,30 @@ func TestCheckRecommendCacheTimeout(t *testing.T) { // create mock worker w := newMockWorker(t) defer w.Close(t) + ctx := context.Background() // empty cache - assert.True(t, w.checkRecommendCacheTimeout("0", nil)) - err := w.CacheClient.SetSorted(cache.Key(cache.OfflineRecommend, "0"), []cache.Scored{{"0", 0}}) + assert.True(t, w.checkRecommendCacheTimeout(ctx, "0", nil)) + err := w.CacheClient.SetSorted(ctx, cache.Key(cache.OfflineRecommend, "0"), []cache.Scored{{"0", 0}}) assert.NoError(t, err) // digest mismatch - assert.True(t, w.checkRecommendCacheTimeout("0", nil)) - err = w.CacheClient.Set(cache.String(cache.Key(cache.OfflineRecommendDigest, "0"), w.Config.OfflineRecommendDigest())) + assert.True(t, w.checkRecommendCacheTimeout(ctx, "0", nil)) + err = w.CacheClient.Set(ctx, cache.String(cache.Key(cache.OfflineRecommendDigest, "0"), w.Config.OfflineRecommendDigest())) assert.NoError(t, err) - err = w.CacheClient.Set(cache.Time(cache.Key(cache.LastModifyUserTime, "0"), time.Now().Add(-time.Hour))) + err = w.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastModifyUserTime, "0"), time.Now().Add(-time.Hour))) assert.NoError(t, err) - assert.True(t, w.checkRecommendCacheTimeout("0", nil)) - err = w.CacheClient.Set(cache.Time(cache.Key(cache.LastUpdateUserRecommendTime, "0"), time.Now().Add(-time.Hour*100))) + assert.True(t, w.checkRecommendCacheTimeout(ctx, "0", nil)) + err = w.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastUpdateUserRecommendTime, "0"), time.Now().Add(-time.Hour*100))) assert.NoError(t, err) - assert.True(t, w.checkRecommendCacheTimeout("0", nil)) - err = w.CacheClient.Set(cache.Time(cache.Key(cache.LastUpdateUserRecommendTime, "0"), time.Now().Add(time.Hour*100))) + assert.True(t, w.checkRecommendCacheTimeout(ctx, "0", nil)) + err = w.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastUpdateUserRecommendTime, "0"), time.Now().Add(time.Hour*100))) assert.NoError(t, err) - assert.False(t, w.checkRecommendCacheTimeout("0", nil)) - err = w.CacheClient.SetSorted(cache.Key(cache.OfflineRecommend, "0"), nil) + assert.False(t, w.checkRecommendCacheTimeout(ctx, "0", nil)) + err = w.CacheClient.SetSorted(ctx, cache.Key(cache.OfflineRecommend, "0"), nil) assert.NoError(t, err) - assert.True(t, w.checkRecommendCacheTimeout("0", nil)) + assert.True(t, w.checkRecommendCacheTimeout(ctx, "0", nil)) } type mockMatrixFactorizationForRecommend struct { @@ -196,11 +199,12 @@ func TestRecommendMatrixFactorizationBruteForce(t *testing.T) { // create mock worker w := newMockWorker(t) defer w.Close(t) + ctx := context.Background() w.Config.Recommend.Offline.EnableColRecommend = true w.Config.Recommend.Collaborative.EnableIndex = false // insert feedbacks now := time.Now() - err := w.DataClient.BatchInsertFeedback([]data.Feedback{ + err := w.DataClient.BatchInsertFeedback(ctx, []data.Feedback{ {FeedbackKey: data.FeedbackKey{FeedbackType: "click", UserId: "0", ItemId: "9"}, Timestamp: now.Add(-time.Hour)}, {FeedbackKey: data.FeedbackKey{FeedbackType: "click", UserId: "0", ItemId: "8"}, Timestamp: now.Add(-time.Hour)}, {FeedbackKey: data.FeedbackKey{FeedbackType: "click", UserId: "0", ItemId: "7"}, Timestamp: now.Add(-time.Hour)}, @@ -215,7 +219,7 @@ func TestRecommendMatrixFactorizationBruteForce(t *testing.T) { assert.NoError(t, err) // insert hidden items and categorized items - err = w.DataClient.BatchInsertItems([]data.Item{ + err = w.DataClient.BatchInsertItems(ctx, []data.Item{ {ItemId: "10", IsHidden: true}, {ItemId: "11", IsHidden: true}, {ItemId: "3", Categories: []string{"*"}}, @@ -227,7 +231,7 @@ func TestRecommendMatrixFactorizationBruteForce(t *testing.T) { w.RankingModel = newMockMatrixFactorizationForRecommend(1, 12) w.Recommend([]data.User{{UserId: "0"}}) - recommends, err := w.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, "0"), 0, -1) + recommends, err := w.CacheClient.GetSorted(ctx, cache.Key(cache.OfflineRecommend, "0"), 0, -1) assert.NoError(t, err) assert.Equal(t, []cache.Scored{ {"3", 3}, @@ -235,14 +239,14 @@ func TestRecommendMatrixFactorizationBruteForce(t *testing.T) { {"1", 1}, {"0", 0}, }, recommends) - recommends, err = w.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, "0", "*"), 0, -1) + recommends, err = w.CacheClient.GetSorted(ctx, cache.Key(cache.OfflineRecommend, "0", "*"), 0, -1) assert.NoError(t, err) assert.Equal(t, []cache.Scored{ {"3", 3}, {"1", 1}, }, recommends) - readCache, err := w.CacheClient.GetSorted(cache.Key(cache.IgnoreItems, "0"), 0, -1) + readCache, err := w.CacheClient.GetSorted(ctx, cache.Key(cache.IgnoreItems, "0"), 0, -1) read := cache.RemoveScores(readCache) assert.NoError(t, err) assert.ElementsMatch(t, []string{"0", "1", "2", "3"}, read) @@ -255,11 +259,13 @@ func TestRecommendMatrixFactorizationHNSW(t *testing.T) { // create mock worker w := newMockWorker(t) defer w.Close(t) + + ctx := context.Background() w.Config.Recommend.Offline.EnableColRecommend = true w.Config.Recommend.Collaborative.EnableIndex = true // insert feedbacks now := time.Now() - err := w.DataClient.BatchInsertFeedback([]data.Feedback{ + err := w.DataClient.BatchInsertFeedback(ctx, []data.Feedback{ {FeedbackKey: data.FeedbackKey{FeedbackType: "click", UserId: "0", ItemId: "9"}, Timestamp: now.Add(-time.Hour)}, {FeedbackKey: data.FeedbackKey{FeedbackType: "click", UserId: "0", ItemId: "8"}, Timestamp: now.Add(-time.Hour)}, {FeedbackKey: data.FeedbackKey{FeedbackType: "click", UserId: "0", ItemId: "7"}, Timestamp: now.Add(-time.Hour)}, @@ -274,7 +280,7 @@ func TestRecommendMatrixFactorizationHNSW(t *testing.T) { assert.NoError(t, err) // insert hidden items and categorized items - err = w.DataClient.BatchInsertItems([]data.Item{ + err = w.DataClient.BatchInsertItems(ctx, []data.Item{ {ItemId: "10", IsHidden: true}, {ItemId: "11", IsHidden: true}, {ItemId: "3", Categories: []string{"*"}}, @@ -286,7 +292,7 @@ func TestRecommendMatrixFactorizationHNSW(t *testing.T) { w.RankingModel = newMockMatrixFactorizationForRecommend(1, 12) w.Recommend([]data.User{{UserId: "0"}}) - recommends, err := w.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, "0"), 0, -1) + recommends, err := w.CacheClient.GetSorted(ctx, cache.Key(cache.OfflineRecommend, "0"), 0, -1) assert.NoError(t, err) assert.Equal(t, []cache.Scored{ {"3", 3}, @@ -294,14 +300,14 @@ func TestRecommendMatrixFactorizationHNSW(t *testing.T) { {"1", 1}, {"0", 0}, }, recommends) - recommends, err = w.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, "0", "*"), 0, -1) + recommends, err = w.CacheClient.GetSorted(ctx, cache.Key(cache.OfflineRecommend, "0", "*"), 0, -1) assert.NoError(t, err) assert.Equal(t, []cache.Scored{ {"3", 3}, {"1", 1}, }, recommends) - readCache, err := w.CacheClient.GetSorted(cache.Key(cache.IgnoreItems, "0"), 0, -1) + readCache, err := w.CacheClient.GetSorted(ctx, cache.Key(cache.IgnoreItems, "0"), 0, -1) read := cache.RemoveScores(readCache) assert.NoError(t, err) assert.ElementsMatch(t, []string{"0", "1", "2", "3"}, read) @@ -314,10 +320,11 @@ func TestRecommend_ItemBased(t *testing.T) { // create mock worker w := newMockWorker(t) defer w.Close(t) + ctx := context.Background() w.Config.Recommend.Offline.EnableColRecommend = false w.Config.Recommend.Offline.EnableItemBasedRecommend = true // insert feedback - err := w.DataClient.BatchInsertFeedback([]data.Feedback{ + err := w.DataClient.BatchInsertFeedback(ctx, []data.Feedback{ {FeedbackKey: data.FeedbackKey{FeedbackType: "a", UserId: "0", ItemId: "21"}}, {FeedbackKey: data.FeedbackKey{FeedbackType: "a", UserId: "0", ItemId: "22"}}, {FeedbackKey: data.FeedbackKey{FeedbackType: "a", UserId: "0", ItemId: "23"}}, @@ -326,20 +333,20 @@ func TestRecommend_ItemBased(t *testing.T) { assert.NoError(t, err) // insert similar items - err = w.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "21"), []cache.Scored{ + err = w.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "21"), []cache.Scored{ {"22", 100000}, {"25", 1000000}, {"29", 1}, }) assert.NoError(t, err) - err = w.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "22"), []cache.Scored{ + err = w.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "22"), []cache.Scored{ {"23", 100000}, {"25", 1000000}, {"28", 1}, {"29", 1}, }) assert.NoError(t, err) - err = w.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "23"), []cache.Scored{ + err = w.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "23"), []cache.Scored{ {"24", 100000}, {"25", 1000000}, {"27", 1}, @@ -347,7 +354,7 @@ func TestRecommend_ItemBased(t *testing.T) { {"29", 1}, }) assert.NoError(t, err) - err = w.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "24"), []cache.Scored{ + err = w.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "24"), []cache.Scored{ {"21", 100000}, {"25", 1000000}, {"26", 1}, @@ -358,41 +365,41 @@ func TestRecommend_ItemBased(t *testing.T) { assert.NoError(t, err) // insert similar items in category - err = w.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "21", "*"), []cache.Scored{ + err = w.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "21", "*"), []cache.Scored{ {"22", 100000}, }) assert.NoError(t, err) - err = w.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "22", "*"), []cache.Scored{ + err = w.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "22", "*"), []cache.Scored{ {"28", 1}, }) assert.NoError(t, err) - err = w.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "23", "*"), []cache.Scored{ + err = w.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "23", "*"), []cache.Scored{ {"24", 100000}, {"28", 1}, }) assert.NoError(t, err) - err = w.CacheClient.SetSorted(cache.Key(cache.ItemNeighbors, "24", "*"), []cache.Scored{ + err = w.CacheClient.SetSorted(ctx, cache.Key(cache.ItemNeighbors, "24", "*"), []cache.Scored{ {"26", 1}, {"28", 1}, }) assert.NoError(t, err) // insert items - err = w.DataClient.BatchInsertItems([]data.Item{{ItemId: "21"}, {ItemId: "22"}, {ItemId: "23"}, {ItemId: "24"}, + err = w.DataClient.BatchInsertItems(ctx, []data.Item{{ItemId: "21"}, {ItemId: "22"}, {ItemId: "23"}, {ItemId: "24"}, {ItemId: "25"}, {ItemId: "26"}, {ItemId: "27"}, {ItemId: "28"}, {ItemId: "29"}}) assert.NoError(t, err) // insert hidden items - err = w.DataClient.BatchInsertItems([]data.Item{{ItemId: "25", IsHidden: true}}) + err = w.DataClient.BatchInsertItems(ctx, []data.Item{{ItemId: "25", IsHidden: true}}) assert.NoError(t, err) // insert categorized items - err = w.DataClient.BatchInsertItems([]data.Item{{ItemId: "26", Categories: []string{"*"}}, {ItemId: "28", Categories: []string{"*"}}}) + err = w.DataClient.BatchInsertItems(ctx, []data.Item{{ItemId: "26", Categories: []string{"*"}}, {ItemId: "28", Categories: []string{"*"}}}) assert.NoError(t, err) w.RankingModel = newMockMatrixFactorizationForRecommend(1, 10) w.Recommend([]data.User{{UserId: "0"}}) - recommends, err := w.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, "0"), 0, 2) + recommends, err := w.CacheClient.GetSorted(ctx, cache.Key(cache.OfflineRecommend, "0"), 0, 2) assert.NoError(t, err) assert.Equal(t, []cache.Scored{{"29", 29}, {"28", 28}, {"27", 27}}, recommends) - recommends, err = w.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, "0", "*"), 0, 2) + recommends, err = w.CacheClient.GetSorted(ctx, cache.Key(cache.OfflineRecommend, "0", "*"), 0, 2) assert.NoError(t, err) assert.Equal(t, []cache.Scored{{"28", 28}, {"26", 26}}, recommends) } @@ -401,48 +408,49 @@ func TestRecommend_UserBased(t *testing.T) { // create mock worker w := newMockWorker(t) defer w.Close(t) + ctx := context.Background() w.Config.Recommend.Offline.EnableColRecommend = false w.Config.Recommend.Offline.EnableUserBasedRecommend = true // insert similar users - err := w.CacheClient.SetSorted(cache.Key(cache.UserNeighbors, "0"), []cache.Scored{ + err := w.CacheClient.SetSorted(ctx, cache.Key(cache.UserNeighbors, "0"), []cache.Scored{ {"1", 2}, {"2", 1.5}, {"3", 1}, }) assert.NoError(t, err) // insert feedback - err = w.DataClient.BatchInsertFeedback([]data.Feedback{ + err = w.DataClient.BatchInsertFeedback(ctx, []data.Feedback{ {FeedbackKey: data.FeedbackKey{FeedbackType: "a", UserId: "1", ItemId: "10"}}, {FeedbackKey: data.FeedbackKey{FeedbackType: "a", UserId: "1", ItemId: "11"}}, }, true, true, true) assert.NoError(t, err) - err = w.DataClient.BatchInsertFeedback([]data.Feedback{ + err = w.DataClient.BatchInsertFeedback(ctx, []data.Feedback{ {FeedbackKey: data.FeedbackKey{FeedbackType: "a", UserId: "2", ItemId: "10"}}, {FeedbackKey: data.FeedbackKey{FeedbackType: "a", UserId: "2", ItemId: "12"}}, {FeedbackKey: data.FeedbackKey{FeedbackType: "a", UserId: "2", ItemId: "48"}}, }, true, true, true) assert.NoError(t, err) - err = w.DataClient.BatchInsertFeedback([]data.Feedback{ + err = w.DataClient.BatchInsertFeedback(ctx, []data.Feedback{ {FeedbackKey: data.FeedbackKey{FeedbackType: "a", UserId: "3", ItemId: "10"}}, {FeedbackKey: data.FeedbackKey{FeedbackType: "a", UserId: "3", ItemId: "13"}}, {FeedbackKey: data.FeedbackKey{FeedbackType: "a", UserId: "3", ItemId: "48"}}, }, true, true, true) assert.NoError(t, err) // insert hidden items - err = w.DataClient.BatchInsertItems([]data.Item{{ItemId: "10", IsHidden: true}}) + err = w.DataClient.BatchInsertItems(ctx, []data.Item{{ItemId: "10", IsHidden: true}}) assert.NoError(t, err) // insert categorized items - err = w.DataClient.BatchInsertItems([]data.Item{ + err = w.DataClient.BatchInsertItems(ctx, []data.Item{ {ItemId: "12", Categories: []string{"*"}}, {ItemId: "48", Categories: []string{"*"}}, }) assert.NoError(t, err) w.RankingModel = newMockMatrixFactorizationForRecommend(1, 10) w.Recommend([]data.User{{UserId: "0"}}) - recommends, err := w.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, "0"), 0, 2) + recommends, err := w.CacheClient.GetSorted(ctx, cache.Key(cache.OfflineRecommend, "0"), 0, 2) assert.NoError(t, err) assert.Equal(t, []cache.Scored{{"48", 48}, {"13", 13}, {"12", 12}}, recommends) - recommends, err = w.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, "0", "*"), 0, 2) + recommends, err = w.CacheClient.GetSorted(ctx, cache.Key(cache.OfflineRecommend, "0", "*"), 0, 2) assert.NoError(t, err) assert.Equal(t, []cache.Scored{{"48", 48}, {"12", 12}}, recommends) } @@ -451,16 +459,18 @@ func TestRecommend_Popular(t *testing.T) { // create mock worker w := newMockWorker(t) defer w.Close(t) + + ctx := context.Background() w.Config.Recommend.Offline.EnableColRecommend = false w.Config.Recommend.Offline.EnablePopularRecommend = true // insert popular items - err := w.CacheClient.SetSorted(cache.PopularItems, []cache.Scored{{"11", 11}, {"10", 10}, {"9", 9}, {"8", 8}}) + err := w.CacheClient.SetSorted(ctx, cache.PopularItems, []cache.Scored{{"11", 11}, {"10", 10}, {"9", 9}, {"8", 8}}) assert.NoError(t, err) // insert popular items with category * - err = w.CacheClient.SetSorted(cache.Key(cache.PopularItems, "*"), []cache.Scored{{"20", 20}, {"19", 19}, {"18", 18}}) + err = w.CacheClient.SetSorted(ctx, cache.Key(cache.PopularItems, "*"), []cache.Scored{{"20", 20}, {"19", 19}, {"18", 18}}) assert.NoError(t, err) // insert items - err = w.DataClient.BatchInsertItems([]data.Item{ + err = w.DataClient.BatchInsertItems(ctx, []data.Item{ {ItemId: "11"}, {ItemId: "10"}, {ItemId: "9"}, {ItemId: "8"}, {ItemId: "20", Categories: []string{"*"}}, {ItemId: "19", Categories: []string{"*"}}, @@ -468,14 +478,14 @@ func TestRecommend_Popular(t *testing.T) { }) assert.NoError(t, err) // insert hidden items - err = w.DataClient.BatchInsertItems([]data.Item{{ItemId: "11", IsHidden: true}}) + err = w.DataClient.BatchInsertItems(ctx, []data.Item{{ItemId: "11", IsHidden: true}}) assert.NoError(t, err) w.RankingModel = newMockMatrixFactorizationForRecommend(1, 10) w.Recommend([]data.User{{UserId: "0"}}) - recommends, err := w.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, "0"), 0, -1) + recommends, err := w.CacheClient.GetSorted(ctx, cache.Key(cache.OfflineRecommend, "0"), 0, -1) assert.NoError(t, err) assert.Equal(t, []cache.Scored{{"10", 10}, {"9", 9}, {"8", 8}}, recommends) - recommends, err = w.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, "0", "*"), 0, -1) + recommends, err = w.CacheClient.GetSorted(ctx, cache.Key(cache.OfflineRecommend, "0", "*"), 0, -1) assert.NoError(t, err) assert.Equal(t, []cache.Scored{{"20", 20}, {"19", 19}, {"18", 18}}, recommends) } @@ -484,16 +494,17 @@ func TestRecommend_Latest(t *testing.T) { // create mock worker w := newMockWorker(t) defer w.Close(t) + ctx := context.Background() w.Config.Recommend.Offline.EnableColRecommend = false w.Config.Recommend.Offline.EnableLatestRecommend = true // insert latest items - err := w.CacheClient.SetSorted(cache.LatestItems, []cache.Scored{{"11", 11}, {"10", 10}, {"9", 9}, {"8", 8}}) + err := w.CacheClient.SetSorted(ctx, cache.LatestItems, []cache.Scored{{"11", 11}, {"10", 10}, {"9", 9}, {"8", 8}}) assert.NoError(t, err) // insert the latest items with category * - err = w.CacheClient.SetSorted(cache.Key(cache.LatestItems, "*"), []cache.Scored{{"20", 10}, {"19", 9}, {"18", 8}}) + err = w.CacheClient.SetSorted(ctx, cache.Key(cache.LatestItems, "*"), []cache.Scored{{"20", 10}, {"19", 9}, {"18", 8}}) assert.NoError(t, err) // insert items - err = w.DataClient.BatchInsertItems([]data.Item{ + err = w.DataClient.BatchInsertItems(ctx, []data.Item{ {ItemId: "11"}, {ItemId: "10"}, {ItemId: "9"}, {ItemId: "8"}, {ItemId: "20", Categories: []string{"*"}}, {ItemId: "19", Categories: []string{"*"}}, @@ -501,14 +512,14 @@ func TestRecommend_Latest(t *testing.T) { }) assert.NoError(t, err) // insert hidden items - err = w.DataClient.BatchInsertItems([]data.Item{{ItemId: "11", IsHidden: true}}) + err = w.DataClient.BatchInsertItems(ctx, []data.Item{{ItemId: "11", IsHidden: true}}) assert.NoError(t, err) w.RankingModel = newMockMatrixFactorizationForRecommend(1, 10) w.Recommend([]data.User{{UserId: "0"}}) - recommends, err := w.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, "0"), 0, -1) + recommends, err := w.CacheClient.GetSorted(ctx, cache.Key(cache.OfflineRecommend, "0"), 0, -1) assert.NoError(t, err) assert.Equal(t, []cache.Scored{{"10", 10}, {"9", 9}, {"8", 8}}, recommends) - recommends, err = w.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, "0", "*"), 0, -1) + recommends, err = w.CacheClient.GetSorted(ctx, cache.Key(cache.OfflineRecommend, "0", "*"), 0, -1) assert.NoError(t, err) assert.Equal(t, []cache.Scored{{"20", 20}, {"19", 19}, {"18", 18}}, recommends) } @@ -517,16 +528,18 @@ func TestRecommend_ColdStart(t *testing.T) { // create mock worker w := newMockWorker(t) defer w.Close(t) + + ctx := context.Background() w.Config.Recommend.Offline.EnableColRecommend = true w.Config.Recommend.Offline.EnableLatestRecommend = true // insert latest items - err := w.CacheClient.SetSorted(cache.LatestItems, []cache.Scored{{"11", 11}, {"10", 10}, {"9", 9}, {"8", 8}}) + err := w.CacheClient.SetSorted(ctx, cache.LatestItems, []cache.Scored{{"11", 11}, {"10", 10}, {"9", 9}, {"8", 8}}) assert.NoError(t, err) // insert the latest items with category * - err = w.CacheClient.SetSorted(cache.Key(cache.LatestItems, "*"), []cache.Scored{{"20", 10}, {"19", 9}, {"18", 8}}) + err = w.CacheClient.SetSorted(ctx, cache.Key(cache.LatestItems, "*"), []cache.Scored{{"20", 10}, {"19", 9}, {"18", 8}}) assert.NoError(t, err) // insert items - err = w.DataClient.BatchInsertItems([]data.Item{ + err = w.DataClient.BatchInsertItems(ctx, []data.Item{ {ItemId: "11"}, {ItemId: "10"}, {ItemId: "9"}, {ItemId: "8"}, {ItemId: "20", Categories: []string{"*"}}, {ItemId: "19", Categories: []string{"*"}}, @@ -534,26 +547,26 @@ func TestRecommend_ColdStart(t *testing.T) { }) assert.NoError(t, err) // insert hidden items - err = w.DataClient.BatchInsertItems([]data.Item{{ItemId: "11", IsHidden: true}}) + err = w.DataClient.BatchInsertItems(ctx, []data.Item{{ItemId: "11", IsHidden: true}}) assert.NoError(t, err) // ranking model not exist m := newMockMatrixFactorizationForRecommend(10, 100) w.Recommend([]data.User{{UserId: "0"}}) - recommends, err := w.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, "0"), 0, -1) + recommends, err := w.CacheClient.GetSorted(ctx, cache.Key(cache.OfflineRecommend, "0"), 0, -1) assert.NoError(t, err) assert.Equal(t, []string{"10", "9", "8"}, cache.RemoveScores(recommends)) - recommends, err = w.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, "0", "*"), 0, -1) + recommends, err = w.CacheClient.GetSorted(ctx, cache.Key(cache.OfflineRecommend, "0", "*"), 0, -1) assert.NoError(t, err) assert.Equal(t, []string{"20", "19", "18"}, cache.RemoveScores(recommends)) // user not predictable w.RankingModel = m w.Recommend([]data.User{{UserId: "100"}}) - recommends, err = w.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, "100"), 0, -1) + recommends, err = w.CacheClient.GetSorted(ctx, cache.Key(cache.OfflineRecommend, "100"), 0, -1) assert.NoError(t, err) assert.Equal(t, []string{"10", "9", "8"}, cache.RemoveScores(recommends)) - recommends, err = w.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, "100", "*"), 0, -1) + recommends, err = w.CacheClient.GetSorted(ctx, cache.Key(cache.OfflineRecommend, "100", "*"), 0, -1) assert.NoError(t, err) assert.Equal(t, []string{"20", "19", "18"}, cache.RemoveScores(recommends)) } @@ -567,12 +580,14 @@ func TestExploreRecommend(t *testing.T) { // create mock worker w := newMockWorker(t) defer w.Close(t) + + ctx := context.Background() w.Config.Recommend.Offline.ExploreRecommend = map[string]float64{"popular": 0.3, "latest": 0.3} // insert popular items - err := w.CacheClient.SetSorted(cache.PopularItems, []cache.Scored{{"popular", 0}}) + err := w.CacheClient.SetSorted(ctx, cache.PopularItems, []cache.Scored{{"popular", 0}}) assert.NoError(t, err) // insert latest items - err = w.CacheClient.SetSorted(cache.LatestItems, []cache.Scored{{"latest", 0}}) + err = w.CacheClient.SetSorted(ctx, cache.LatestItems, []cache.Scored{{"latest", 0}}) assert.NoError(t, err) recommend, err := w.exploreRecommend(cache.CreateScoredItems( @@ -788,8 +803,10 @@ func TestRankByCollaborativeFiltering(t *testing.T) { // create mock worker w := newMockWorker(t) defer w.Close(t) + + ctx := context.Background() // insert a user - err := w.DataClient.BatchInsertUsers([]data.User{{UserId: "1"}}) + err := w.DataClient.BatchInsertUsers(ctx, []data.User{{UserId: "1"}}) assert.NoError(t, err) // insert items itemCache := make(map[string]data.Item) @@ -808,8 +825,10 @@ func TestRankByClickTroughRate(t *testing.T) { // create mock worker w := newMockWorker(t) defer w.Close(t) + + ctx := context.Background() // insert a user - err := w.DataClient.BatchInsertUsers([]data.User{{UserId: "1"}}) + err := w.DataClient.BatchInsertUsers(ctx, []data.User{{UserId: "1"}}) assert.NoError(t, err) // insert items itemCache := NewItemCache() @@ -828,6 +847,8 @@ func TestReplacement_ClickThroughRate(t *testing.T) { // create mock worker w := newMockWorker(t) defer w.Close(t) + + ctx := context.Background() w.Config.Recommend.DataSource.PositiveFeedbackTypes = []string{"p"} w.Config.Recommend.DataSource.ReadFeedbackTypes = []string{"n"} w.Config.Recommend.Offline.EnableColRecommend = false @@ -837,12 +858,12 @@ func TestReplacement_ClickThroughRate(t *testing.T) { // 1. Insert historical items into empty recommendation. // insert items - err := w.DataClient.BatchInsertItems([]data.Item{ + err := w.DataClient.BatchInsertItems(ctx, []data.Item{ {ItemId: "10"}, {ItemId: "9"}, {ItemId: "8"}, {ItemId: "7"}, {ItemId: "6"}, {ItemId: "5"}, }) assert.NoError(t, err) // insert feedback - err = w.DataClient.BatchInsertFeedback([]data.Feedback{ + err = w.DataClient.BatchInsertFeedback(ctx, []data.Feedback{ {FeedbackKey: data.FeedbackKey{FeedbackType: "p", UserId: "0", ItemId: "10"}}, {FeedbackKey: data.FeedbackKey{FeedbackType: "n", UserId: "0", ItemId: "9"}}, {FeedbackKey: data.FeedbackKey{FeedbackType: "i", UserId: "0", ItemId: "8"}}, @@ -850,25 +871,25 @@ func TestReplacement_ClickThroughRate(t *testing.T) { assert.NoError(t, err) w.ClickModel = new(mockFactorizationMachine) w.Recommend([]data.User{{UserId: "0"}}) - recommends, err := w.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, "0"), 0, 2) + recommends, err := w.CacheClient.GetSorted(ctx, cache.Key(cache.OfflineRecommend, "0"), 0, 2) assert.NoError(t, err) assert.Equal(t, []cache.Scored{{"10", 10}, {"9", 9}}, recommends) // 2. Insert historical items into non-empty recommendation. - err = w.CacheClient.Set(cache.Time(cache.Key(cache.LastUpdateUserRecommendTime, "0"), time.Now().AddDate(-1, 0, 0))) + err = w.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastUpdateUserRecommendTime, "0"), time.Now().AddDate(-1, 0, 0))) assert.NoError(t, err) // insert popular items - err = w.CacheClient.SetSorted(cache.PopularItems, []cache.Scored{{"7", 10}, {"6", 9}, {"5", 8}}) + err = w.CacheClient.SetSorted(ctx, cache.PopularItems, []cache.Scored{{"7", 10}, {"6", 9}, {"5", 8}}) assert.NoError(t, err) // insert feedback - err = w.DataClient.BatchInsertFeedback([]data.Feedback{ + err = w.DataClient.BatchInsertFeedback(ctx, []data.Feedback{ {FeedbackKey: data.FeedbackKey{FeedbackType: "p", UserId: "0", ItemId: "10"}}, {FeedbackKey: data.FeedbackKey{FeedbackType: "n", UserId: "0", ItemId: "9"}}, {FeedbackKey: data.FeedbackKey{FeedbackType: "i", UserId: "0", ItemId: "8"}}, }, true, false, true) assert.NoError(t, err) w.Recommend([]data.User{{UserId: "0"}}) - recommends, err = w.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, "0"), 0, 2) + recommends, err = w.CacheClient.GetSorted(ctx, cache.Key(cache.OfflineRecommend, "0"), 0, 2) assert.NoError(t, err) assert.Equal(t, []cache.Scored{{"10", 9}, {"9", 7.4}, {"7", 7}}, recommends) } @@ -877,6 +898,8 @@ func TestReplacement_CollaborativeFiltering(t *testing.T) { // create mock worker w := newMockWorker(t) defer w.Close(t) + + ctx := context.Background() w.Config.Recommend.DataSource.PositiveFeedbackTypes = []string{"p"} w.Config.Recommend.DataSource.ReadFeedbackTypes = []string{"n"} w.Config.Recommend.Offline.EnableColRecommend = false @@ -885,12 +908,12 @@ func TestReplacement_CollaborativeFiltering(t *testing.T) { // 1. Insert historical items into empty recommendation. // insert items - err := w.DataClient.BatchInsertItems([]data.Item{ + err := w.DataClient.BatchInsertItems(ctx, []data.Item{ {ItemId: "10"}, {ItemId: "9"}, {ItemId: "8"}, {ItemId: "7"}, {ItemId: "6"}, {ItemId: "5"}, }) assert.NoError(t, err) // insert feedback - err = w.DataClient.BatchInsertFeedback([]data.Feedback{ + err = w.DataClient.BatchInsertFeedback(ctx, []data.Feedback{ {FeedbackKey: data.FeedbackKey{FeedbackType: "p", UserId: "0", ItemId: "10"}}, {FeedbackKey: data.FeedbackKey{FeedbackType: "n", UserId: "0", ItemId: "9"}}, {FeedbackKey: data.FeedbackKey{FeedbackType: "i", UserId: "0", ItemId: "8"}}, @@ -898,25 +921,25 @@ func TestReplacement_CollaborativeFiltering(t *testing.T) { assert.NoError(t, err) w.RankingModel = newMockMatrixFactorizationForRecommend(1, 10) w.Recommend([]data.User{{UserId: "0"}}) - recommends, err := w.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, "0"), 0, 2) + recommends, err := w.CacheClient.GetSorted(ctx, cache.Key(cache.OfflineRecommend, "0"), 0, 2) assert.NoError(t, err) assert.Equal(t, []cache.Scored{{"10", 10}, {"9", 9}}, recommends) // 2. Insert historical items into non-empty recommendation. - err = w.CacheClient.Set(cache.Time(cache.Key(cache.LastUpdateUserRecommendTime, "0"), time.Now().AddDate(-1, 0, 0))) + err = w.CacheClient.Set(ctx, cache.Time(cache.Key(cache.LastUpdateUserRecommendTime, "0"), time.Now().AddDate(-1, 0, 0))) assert.NoError(t, err) // insert popular items - err = w.CacheClient.SetSorted(cache.PopularItems, []cache.Scored{{"7", 10}, {"6", 9}, {"5", 8}}) + err = w.CacheClient.SetSorted(ctx, cache.PopularItems, []cache.Scored{{"7", 10}, {"6", 9}, {"5", 8}}) assert.NoError(t, err) // insert feedback - err = w.DataClient.BatchInsertFeedback([]data.Feedback{ + err = w.DataClient.BatchInsertFeedback(ctx, []data.Feedback{ {FeedbackKey: data.FeedbackKey{FeedbackType: "p", UserId: "0", ItemId: "10"}}, {FeedbackKey: data.FeedbackKey{FeedbackType: "n", UserId: "0", ItemId: "9"}}, {FeedbackKey: data.FeedbackKey{FeedbackType: "i", UserId: "0", ItemId: "8"}}, }, true, false, true) assert.NoError(t, err) w.Recommend([]data.User{{UserId: "0"}}) - recommends, err = w.CacheClient.GetSorted(cache.Key(cache.OfflineRecommend, "0"), 0, 2) + recommends, err = w.CacheClient.GetSorted(ctx, cache.Key(cache.OfflineRecommend, "0"), 0, 2) assert.NoError(t, err) assert.Equal(t, []cache.Scored{{"10", 9}, {"9", 7.4}, {"7", 7}}, recommends) }