From 6ceed2c01713bcc2e158e73939fde7eb016e6da5 Mon Sep 17 00:00:00 2001 From: Nick Zelei <2420177+nickzelei@users.noreply.github.com> Date: Sat, 20 Apr 2024 16:35:28 -0700 Subject: [PATCH] Updates logging tags, transform float/int to use relative min/max (#1798) --- backend/internal/auth/jwt/client.go | 5 +- .../internal/cmds/mgmt/serve/connect/cmd.go | 6 +- .../interceptors/auth_logging/interceptor.go | 69 ++++++ .../auth_logging/interceptor_test.go | 198 ++++++++++++++++++ .../interceptors/logger/interceptor.go | 4 +- .../connect/interceptors/logger/logger-ctx.go | 2 +- .../interceptors/logger/logger-ctx_test.go | 2 +- .../interceptors/logging/interceptor.go | 33 +-- .../interceptors/logging/interceptor_test.go | 13 +- backend/pkg/sqlconnect/sql-connector.go | 8 +- .../v1alpha1/connection-service/connection.go | 14 +- .../transformers/{system.mdx => system.md} | 30 ++- .../UserDefinedGenerateFloat64Form.tsx | 2 +- .../UserDefinedGenerateInt64Form.tsx | 2 +- .../UserDefinedTransformFloat64Form.tsx | 20 +- .../UserDefinedTransformInt64Form.tsx | 20 +- .../Sheetforms/GenerateFloat64Form.tsx | 2 +- .../Sheetforms/GenerateInt64Form.tsx | 2 +- .../Sheetforms/TransformFloat64Form.tsx | 8 +- .../Sheetforms/TransformInt64Form.tsx | 8 +- .../benthos/transformers/transform_float.go | 22 +- .../transformers/transform_float_test.go | 36 ++-- .../benthos/transformers/transform_int64.go | 23 +- .../transformers/transform_int64_test.go | 40 ++-- .../benthos/transformers/utils/float_utils.go | 13 -- .../transformers/utils/float_utils_test.go | 18 -- .../transformers/utils/integer_utils.go | 12 -- .../transformers/utils/integer_utils_test.go | 18 -- 28 files changed, 399 insertions(+), 231 deletions(-) create mode 100644 backend/internal/connect/interceptors/auth_logging/interceptor.go create mode 100644 backend/internal/connect/interceptors/auth_logging/interceptor_test.go rename docs/docs/transformers/{system.mdx => system.md} (97%) diff --git a/backend/internal/auth/jwt/client.go b/backend/internal/auth/jwt/client.go index 2a8e2dbce..a0d4939c6 100644 --- a/backend/internal/auth/jwt/client.go +++ b/backend/internal/auth/jwt/client.go @@ -160,9 +160,10 @@ func getCombinedScopesAndPermissions(scope string, permissions []string) []strin } func GetTokenDataFromCtx(ctx context.Context) (*TokenContextData, error) { - data, ok := ctx.Value(TokenContextKey{}).(*TokenContextData) + val := ctx.Value(TokenContextKey{}) + data, ok := val.(*TokenContextData) if !ok { - return nil, nucleuserrors.NewUnauthenticated("ctx does not contain TokenContextData or unable to cast struct") + return nil, nucleuserrors.NewUnauthenticated(fmt.Sprintf("ctx does not contain TokenContextData or unable to cast struct: %T", val)) } return data, nil } diff --git a/backend/internal/cmds/mgmt/serve/connect/cmd.go b/backend/internal/cmds/mgmt/serve/connect/cmd.go index 5f8ac6d28..de6289beb 100644 --- a/backend/internal/cmds/mgmt/serve/connect/cmd.go +++ b/backend/internal/cmds/mgmt/serve/connect/cmd.go @@ -31,6 +31,7 @@ import ( awsmanager "github.com/nucleuscloud/neosync/backend/internal/aws" up_cmd "github.com/nucleuscloud/neosync/backend/internal/cmds/mgmt/migrate/up" auth_interceptor "github.com/nucleuscloud/neosync/backend/internal/connect/interceptors/auth" + authlogging_interceptor "github.com/nucleuscloud/neosync/backend/internal/connect/interceptors/auth_logging" logger_interceptor "github.com/nucleuscloud/neosync/backend/internal/connect/interceptors/logger" logging_interceptor "github.com/nucleuscloud/neosync/backend/internal/connect/interceptors/logging" neosynclogger "github.com/nucleuscloud/neosync/backend/internal/logger" @@ -158,7 +159,7 @@ func serve(ctx context.Context) error { return err } loggerInterceptor := logger_interceptor.NewInterceptor(logger) - loggingInterceptor := logging_interceptor.NewInterceptor(logger) + loggingInterceptor := logging_interceptor.NewInterceptor() stdInterceptors := []connect.Interceptor{ otelInterceptor, @@ -202,12 +203,14 @@ func serve(ctx context.Context) error { apikeyClient, ).InjectTokenCtx, ), + authlogging_interceptor.NewInterceptor(db), ) jwtOnlyAuthInterceptors = append( jwtOnlyAuthInterceptors, auth_interceptor.NewInterceptor( jwtclient.InjectTokenCtx, ), + authlogging_interceptor.NewInterceptor(db), ) authSvcInterceptors = append( authSvcInterceptors, @@ -221,6 +224,7 @@ func serve(ctx context.Context) error { mgmtv1alpha1connect.AuthServiceRefreshCliProcedure, }, ), + authlogging_interceptor.NewInterceptor(db), ) } diff --git a/backend/internal/connect/interceptors/auth_logging/interceptor.go b/backend/internal/connect/interceptors/auth_logging/interceptor.go new file mode 100644 index 000000000..26611ac4d --- /dev/null +++ b/backend/internal/connect/interceptors/auth_logging/interceptor.go @@ -0,0 +1,69 @@ +package authlogging_interceptor + +import ( + "context" + + "connectrpc.com/connect" + "github.com/nucleuscloud/neosync/backend/internal/auth/tokenctx" + logger_interceptor "github.com/nucleuscloud/neosync/backend/internal/connect/interceptors/logger" + "github.com/nucleuscloud/neosync/backend/internal/nucleusdb" +) + +type Interceptor struct { + db *nucleusdb.NucleusDb +} + +func NewInterceptor(db *nucleusdb.NucleusDb) connect.Interceptor { + return &Interceptor{db: db} +} + +func (i *Interceptor) WrapUnary(next connect.UnaryFunc) connect.UnaryFunc { + return func(ctx context.Context, request connect.AnyRequest) (connect.AnyResponse, error) { + return next(setAuthValues(ctx, i.db), request) + } +} + +func (i *Interceptor) WrapStreamingClient(next connect.StreamingClientFunc) connect.StreamingClientFunc { + return func(ctx context.Context, spec connect.Spec) connect.StreamingClientConn { + return next(ctx, spec) + } +} + +func (i *Interceptor) WrapStreamingHandler(next connect.StreamingHandlerFunc) connect.StreamingHandlerFunc { + return func(ctx context.Context, conn connect.StreamingHandlerConn) error { + return next(setAuthValues(ctx, i.db), conn) + } +} + +func setAuthValues(ctx context.Context, db *nucleusdb.NucleusDb) context.Context { + vals := getAuthValues(ctx, db) + logger := logger_interceptor.GetLoggerFromContextOrDefault(ctx).With(vals...) + return logger_interceptor.SetLoggerContext(ctx, logger) +} + +func getAuthValues(ctx context.Context, db *nucleusdb.NucleusDb) []any { + tokenCtxResp, err := tokenctx.GetTokenCtx(ctx) + if err != nil { + return []any{} + } + output := []any{} + + if tokenCtxResp.JwtContextData != nil { + output = append(output, "authUserId", tokenCtxResp.JwtContextData.AuthUserId) + + user, err := db.Q.GetUserByProviderSub(ctx, db.Db, tokenCtxResp.JwtContextData.AuthUserId) + if err == nil { + output = append(output, "userId", nucleusdb.UUIDString(user.ID)) + } + } else if tokenCtxResp.ApiKeyContextData != nil { + output = append(output, "apiKeyType", tokenCtxResp.ApiKeyContextData.ApiKeyType) + if tokenCtxResp.ApiKeyContextData.ApiKey != nil { + output = append(output, + "apiKeyId", nucleusdb.UUIDString(tokenCtxResp.ApiKeyContextData.ApiKey.ID), + "accountId", nucleusdb.UUIDString(tokenCtxResp.ApiKeyContextData.ApiKey.AccountID), + "userId", nucleusdb.UUIDString(tokenCtxResp.ApiKeyContextData.ApiKey.UserID), + ) + } + } + return output +} diff --git a/backend/internal/connect/interceptors/auth_logging/interceptor_test.go b/backend/internal/connect/interceptors/auth_logging/interceptor_test.go new file mode 100644 index 000000000..dd88abc7e --- /dev/null +++ b/backend/internal/connect/interceptors/auth_logging/interceptor_test.go @@ -0,0 +1,198 @@ +package authlogging_interceptor + +import ( + "context" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "testing" + + "connectrpc.com/connect" + "github.com/google/uuid" + db_queries "github.com/nucleuscloud/neosync/backend/gen/go/db" + mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" + "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1/mgmtv1alpha1connect" + "github.com/nucleuscloud/neosync/backend/internal/apikey" + auth_apikey "github.com/nucleuscloud/neosync/backend/internal/auth/apikey" + auth_jwt "github.com/nucleuscloud/neosync/backend/internal/auth/jwt" + logger_interceptor "github.com/nucleuscloud/neosync/backend/internal/connect/interceptors/logger" + "github.com/nucleuscloud/neosync/backend/internal/nucleusdb" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func Test_Interceptor_WrapUnary_JwtContextData_ValidUser(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + + mockDbtx := nucleusdb.NewMockDBTX(t) + mockQuerier := db_queries.NewMockQuerier(t) + + genuuid, _ := nucleusdb.ToUuid(uuid.NewString()) + mockQuerier.On("GetUserByProviderSub", mock.Anything, mock.Anything, "auth-user-id"). + Return(db_queries.NeosyncApiUser{ID: genuuid}, nil) + + mux := http.NewServeMux() + mux.Handle(mgmtv1alpha1connect.UserAccountServiceGetUserProcedure, connect.NewUnaryHandler( + mgmtv1alpha1connect.UserAccountServiceGetUserProcedure, + func(ctx context.Context, r *connect.Request[mgmtv1alpha1.GetUserRequest]) (*connect.Response[mgmtv1alpha1.GetUserResponse], error) { + return connect.NewResponse(&mgmtv1alpha1.GetUserResponse{UserId: "123"}), nil + }, + connect.WithInterceptors( + logger_interceptor.NewInterceptor(logger), + &mockAuthInterceptor{data: &auth_jwt.TokenContextData{AuthUserId: "auth-user-id"}}, + NewInterceptor(nucleusdb.New(mockDbtx, mockQuerier)), + ), + )) + + srv := startHTTPServer(t, mux) + client := mgmtv1alpha1connect.NewUserAccountServiceClient(srv.Client(), srv.URL) + _, err := client.GetUser(context.Background(), connect.NewRequest(&mgmtv1alpha1.GetUserRequest{})) + require.NoError(t, err) +} + +func Test_Interceptor_WrapUnary_JwtContextData_NoUser_NoFail(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + + mockDbtx := nucleusdb.NewMockDBTX(t) + mockQuerier := db_queries.NewMockQuerier(t) + + mux := http.NewServeMux() + mux.Handle(mgmtv1alpha1connect.UserAccountServiceGetUserProcedure, connect.NewUnaryHandler( + mgmtv1alpha1connect.UserAccountServiceGetUserProcedure, + func(ctx context.Context, r *connect.Request[mgmtv1alpha1.GetUserRequest]) (*connect.Response[mgmtv1alpha1.GetUserResponse], error) { + return connect.NewResponse(&mgmtv1alpha1.GetUserResponse{UserId: "123"}), nil + }, + connect.WithInterceptors( + logger_interceptor.NewInterceptor(logger), + NewInterceptor(nucleusdb.New(mockDbtx, mockQuerier)), + ), + )) + + srv := startHTTPServer(t, mux) + client := mgmtv1alpha1connect.NewUserAccountServiceClient(srv.Client(), srv.URL) + _, err := client.GetUser(context.Background(), connect.NewRequest(&mgmtv1alpha1.GetUserRequest{})) + require.NoError(t, err) +} + +type mockAuthInterceptor struct { + data *auth_jwt.TokenContextData +} + +func (i *mockAuthInterceptor) WrapUnary(next connect.UnaryFunc) connect.UnaryFunc { + return func(ctx context.Context, request connect.AnyRequest) (connect.AnyResponse, error) { + return next(context.WithValue(ctx, auth_jwt.TokenContextKey{}, i.data), request) + } +} + +func (i *mockAuthInterceptor) WrapStreamingClient(next connect.StreamingClientFunc) connect.StreamingClientFunc { + return func(ctx context.Context, spec connect.Spec) connect.StreamingClientConn { + return next(ctx, spec) + } +} + +func (i *mockAuthInterceptor) WrapStreamingHandler(next connect.StreamingHandlerFunc) connect.StreamingHandlerFunc { + return func(ctx context.Context, conn connect.StreamingHandlerConn) error { + return next(ctx, conn) + } +} + +func startHTTPServer(tb testing.TB, h http.Handler) *httptest.Server { + tb.Helper() + srv := httptest.NewUnstartedServer(h) + srv.EnableHTTP2 = true + srv.Start() + tb.Cleanup(srv.Close) + return srv +} + +func Test_getAuthValues_NoTokenCtx(t *testing.T) { + vals := getAuthValues(context.Background(), &nucleusdb.NucleusDb{}) + require.Empty(t, vals) +} + +func Test_getAuthValues_Valid_Jwt(t *testing.T) { + mockDbtx := nucleusdb.NewMockDBTX(t) + mockQuerier := db_queries.NewMockQuerier(t) + + uuidstr := uuid.NewString() + genuuid, _ := nucleusdb.ToUuid(uuidstr) + mockQuerier.On("GetUserByProviderSub", mock.Anything, mock.Anything, "auth-user-id"). + Return(db_queries.NeosyncApiUser{ID: genuuid}, nil) + + ctx := context.WithValue(context.Background(), auth_jwt.TokenContextKey{}, &auth_jwt.TokenContextData{ + AuthUserId: "auth-user-id", + }) + + vals := getAuthValues(ctx, nucleusdb.New(mockDbtx, mockQuerier)) + require.Equal( + t, + []any{"authUserId", "auth-user-id", "userId", uuidstr}, + vals, + ) +} + +func Test_getAuthValues_Valid_Jwt_No_User(t *testing.T) { + mockDbtx := nucleusdb.NewMockDBTX(t) + mockQuerier := db_queries.NewMockQuerier(t) + + mockQuerier.On("GetUserByProviderSub", mock.Anything, mock.Anything, "auth-user-id"). + Return(db_queries.NeosyncApiUser{}, errors.New("test err")) + + ctx := context.WithValue(context.Background(), auth_jwt.TokenContextKey{}, &auth_jwt.TokenContextData{ + AuthUserId: "auth-user-id", + }) + + vals := getAuthValues(ctx, nucleusdb.New(mockDbtx, mockQuerier)) + require.Equal( + t, + []any{"authUserId", "auth-user-id"}, + vals, + ) +} + +func Test_getAuthValues_Valid_ApiKey(t *testing.T) { + mockDbtx := nucleusdb.NewMockDBTX(t) + mockQuerier := db_queries.NewMockQuerier(t) + + apikeyid := uuid.NewString() + accountid := uuid.NewString() + userid := uuid.NewString() + + apikeyuuid, _ := nucleusdb.ToUuid(apikeyid) + accountiduuid, _ := nucleusdb.ToUuid(accountid) + useriduuid, _ := nucleusdb.ToUuid(userid) + + ctx := context.WithValue(context.Background(), auth_apikey.TokenContextKey{}, &auth_apikey.TokenContextData{ + ApiKeyType: apikey.AccountApiKey, + ApiKey: &db_queries.NeosyncApiAccountApiKey{ + ID: apikeyuuid, + AccountID: accountiduuid, + UserID: useriduuid, + }, + }) + + vals := getAuthValues(ctx, nucleusdb.New(mockDbtx, mockQuerier)) + require.Equal( + t, + []any{"apiKeyType", apikey.AccountApiKey, "apiKeyId", apikeyid, "accountId", accountid, "userId", userid}, + vals, + ) +} + +func Test_getAuthValues_Valid_ApiKey_No_Apikey(t *testing.T) { + mockDbtx := nucleusdb.NewMockDBTX(t) + mockQuerier := db_queries.NewMockQuerier(t) + + ctx := context.WithValue(context.Background(), auth_apikey.TokenContextKey{}, &auth_apikey.TokenContextData{ + ApiKeyType: apikey.AccountApiKey, + }) + + vals := getAuthValues(ctx, nucleusdb.New(mockDbtx, mockQuerier)) + require.Equal( + t, + []any{"apiKeyType", apikey.AccountApiKey}, + vals, + ) +} diff --git a/backend/internal/connect/interceptors/logger/interceptor.go b/backend/internal/connect/interceptors/logger/interceptor.go index 4601945e1..76ca1ef24 100644 --- a/backend/internal/connect/interceptors/logger/interceptor.go +++ b/backend/internal/connect/interceptors/logger/interceptor.go @@ -19,7 +19,7 @@ func NewInterceptor(logger *slog.Logger) connect.Interceptor { func (i *Interceptor) WrapUnary(next connect.UnaryFunc) connect.UnaryFunc { return func(ctx context.Context, request connect.AnyRequest) (connect.AnyResponse, error) { - newCtx := setLoggerContext(ctx, clonelogger(i.logger)) + newCtx := SetLoggerContext(ctx, clonelogger(i.logger)) return next(newCtx, request) } } @@ -32,7 +32,7 @@ func (i *Interceptor) WrapStreamingClient(next connect.StreamingClientFunc) conn func (i *Interceptor) WrapStreamingHandler(next connect.StreamingHandlerFunc) connect.StreamingHandlerFunc { return func(ctx context.Context, conn connect.StreamingHandlerConn) error { - newCtx := setLoggerContext(ctx, clonelogger(i.logger)) + newCtx := SetLoggerContext(ctx, clonelogger(i.logger)) return next(newCtx, conn) } } diff --git a/backend/internal/connect/interceptors/logger/logger-ctx.go b/backend/internal/connect/interceptors/logger/logger-ctx.go index f258fc985..83add30fb 100644 --- a/backend/internal/connect/interceptors/logger/logger-ctx.go +++ b/backend/internal/connect/interceptors/logger/logger-ctx.go @@ -22,6 +22,6 @@ func GetLoggerFromContextOrDefault(ctx context.Context) *slog.Logger { return data.GetLogger() } -func setLoggerContext(ctx context.Context, logger *slog.Logger) context.Context { +func SetLoggerContext(ctx context.Context, logger *slog.Logger) context.Context { return context.WithValue(ctx, loggerContextKey{}, &loggerContextData{logger: logger}) } diff --git a/backend/internal/connect/interceptors/logger/logger-ctx_test.go b/backend/internal/connect/interceptors/logger/logger-ctx_test.go index 110eb2b43..1f6a50264 100644 --- a/backend/internal/connect/interceptors/logger/logger-ctx_test.go +++ b/backend/internal/connect/interceptors/logger/logger-ctx_test.go @@ -15,7 +15,7 @@ func Test_GetLoggerFromContextOrDefault(t *testing.T) { func Test_GetLoggerFromContextOrDefault_NonDefault(t *testing.T) { logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) - ctx := setLoggerContext(context.Background(), logger) + ctx := SetLoggerContext(context.Background(), logger) ctxlogger := GetLoggerFromContextOrDefault(ctx) assert.Equal(t, logger, ctxlogger) } diff --git a/backend/internal/connect/interceptors/logging/interceptor.go b/backend/internal/connect/interceptors/logging/interceptor.go index 548f0c6d4..4e72a436b 100644 --- a/backend/internal/connect/interceptors/logging/interceptor.go +++ b/backend/internal/connect/interceptors/logging/interceptor.go @@ -3,28 +3,30 @@ package logging_interceptor import ( "context" "fmt" - "log/slog" "time" "connectrpc.com/connect" + logger_interceptor "github.com/nucleuscloud/neosync/backend/internal/connect/interceptors/logger" ) type Interceptor struct { - logger *slog.Logger } -func NewInterceptor(logger *slog.Logger) connect.Interceptor { - return &Interceptor{logger: logger} +func NewInterceptor() connect.Interceptor { + return &Interceptor{} } func (i *Interceptor) WrapUnary(next connect.UnaryFunc) connect.UnaryFunc { return func(ctx context.Context, request connect.AnyRequest) (connect.AnyResponse, error) { + logger := logger_interceptor.GetLoggerFromContextOrDefault(ctx) + logger = logger.With("procedure", request.Spec().Procedure) + ctx = logger_interceptor.SetLoggerContext(ctx, logger) + now := time.Now() - i.logger.Info( + logger.Info( "started call", "time_ms", fmt.Sprintf("%d", now.UnixMilli()), "stream_type", request.Spec().StreamType.String(), - "procedure", request.Spec().Procedure, "http_method", request.HTTPMethod(), "peer_protocol", request.Peer().Protocol, ) @@ -34,7 +36,6 @@ func (i *Interceptor) WrapUnary(next connect.UnaryFunc) connect.UnaryFunc { "time_ms", fmt.Sprintf("%d", endNow.UnixMilli()), "duration_ms", fmt.Sprintf("%d", endNow.Sub(now).Milliseconds()), "stream_type", request.Spec().StreamType.String(), - "procedure", request.Spec().Procedure, "http_method", request.HTTPMethod(), "peer_protocol", request.Peer().Protocol, } @@ -48,16 +49,16 @@ func (i *Interceptor) WrapUnary(next connect.UnaryFunc) connect.UnaryFunc { fields = append(fields, "connect.code", connect.CodeInternal.String()) } - i.logger.Error(err.Error(), fields...) + logger.Error(err.Error(), fields...) - i.logger.Info( + logger.Info( "finished call", fields..., ) return nil, err } fields = append(fields, "connect.code", "ok") - i.logger.Info( + logger.Info( "finished call", fields..., ) @@ -73,12 +74,15 @@ func (i *Interceptor) WrapStreamingClient(next connect.StreamingClientFunc) conn func (i *Interceptor) WrapStreamingHandler(next connect.StreamingHandlerFunc) connect.StreamingHandlerFunc { return func(ctx context.Context, conn connect.StreamingHandlerConn) error { + logger := logger_interceptor.GetLoggerFromContextOrDefault(ctx) + logger = logger.With("procedure", conn.Spec().Procedure) + ctx = logger_interceptor.SetLoggerContext(ctx, logger) + now := time.Now() - i.logger.Info( + logger.Info( "started call", "time_ms", fmt.Sprintf("%d", now.UnixMilli()), "stream_type", conn.Spec().StreamType.String(), - "procedure", conn.Spec().Procedure, "peer_protocol", conn.Peer().Protocol, ) err := next(ctx, conn) @@ -87,7 +91,6 @@ func (i *Interceptor) WrapStreamingHandler(next connect.StreamingHandlerFunc) co "time_ms", fmt.Sprintf("%d", endNow.UnixMilli()), "duration_ms", fmt.Sprintf("%d", endNow.Sub(now).Milliseconds()), "stream_type", conn.Spec().StreamType.String(), - "procedure", conn.Spec().Procedure, "peer_protocol", conn.Peer().Protocol, } if err != nil { @@ -97,14 +100,14 @@ func (i *Interceptor) WrapStreamingHandler(next connect.StreamingHandlerFunc) co fields = append(fields, "connect.code", connectErr.Code().String()) } - i.logger.Info( + logger.Info( "finished call", fields..., ) return err } fields = append(fields, "connect.code", "ok") - i.logger.Info( + logger.Info( "finished call", fields..., ) diff --git a/backend/internal/connect/interceptors/logging/interceptor_test.go b/backend/internal/connect/interceptors/logging/interceptor_test.go index 00256ba8e..009726ccf 100644 --- a/backend/internal/connect/interceptors/logging/interceptor_test.go +++ b/backend/internal/connect/interceptors/logging/interceptor_test.go @@ -12,12 +12,13 @@ import ( "connectrpc.com/connect" mgmtv1alpha1 "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1" "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1/mgmtv1alpha1connect" + logger_interceptor "github.com/nucleuscloud/neosync/backend/internal/connect/interceptors/logger" "github.com/stretchr/testify/assert" ) func Test_Interceptor_WrapUnary_Without_Error(t *testing.T) { logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) - interceptor := NewInterceptor(logger) + interceptor := NewInterceptor() mux := http.NewServeMux() mux.Handle(mgmtv1alpha1connect.UserAccountServiceGetUserProcedure, connect.NewUnaryHandler( @@ -25,7 +26,7 @@ func Test_Interceptor_WrapUnary_Without_Error(t *testing.T) { func(ctx context.Context, r *connect.Request[mgmtv1alpha1.GetUserRequest]) (*connect.Response[mgmtv1alpha1.GetUserResponse], error) { return connect.NewResponse(&mgmtv1alpha1.GetUserResponse{UserId: "123"}), nil }, - connect.WithInterceptors(interceptor), + connect.WithInterceptors(logger_interceptor.NewInterceptor(logger), interceptor), )) srv := startHTTPServer(t, mux) @@ -36,7 +37,7 @@ func Test_Interceptor_WrapUnary_Without_Error(t *testing.T) { func Test_Interceptor_WrapUnary_With_Generic_Error(t *testing.T) { logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) - interceptor := NewInterceptor(logger) + interceptor := NewInterceptor() mux := http.NewServeMux() mux.Handle(mgmtv1alpha1connect.UserAccountServiceGetUserProcedure, connect.NewUnaryHandler( @@ -44,7 +45,7 @@ func Test_Interceptor_WrapUnary_With_Generic_Error(t *testing.T) { func(ctx context.Context, r *connect.Request[mgmtv1alpha1.GetUserRequest]) (*connect.Response[mgmtv1alpha1.GetUserResponse], error) { return nil, errors.New("test") }, - connect.WithInterceptors(interceptor), + connect.WithInterceptors(logger_interceptor.NewInterceptor(logger), interceptor), )) srv := startHTTPServer(t, mux) @@ -55,7 +56,7 @@ func Test_Interceptor_WrapUnary_With_Generic_Error(t *testing.T) { func Test_Interceptor_WrapUnary_With_Connect_Error(t *testing.T) { logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) - interceptor := NewInterceptor(logger) + interceptor := NewInterceptor() mux := http.NewServeMux() mux.Handle(mgmtv1alpha1connect.UserAccountServiceGetUserProcedure, connect.NewUnaryHandler( @@ -63,7 +64,7 @@ func Test_Interceptor_WrapUnary_With_Connect_Error(t *testing.T) { func(ctx context.Context, r *connect.Request[mgmtv1alpha1.GetUserRequest]) (*connect.Response[mgmtv1alpha1.GetUserResponse], error) { return nil, connect.NewError(connect.CodeNotFound, errors.New("test")) }, - connect.WithInterceptors(interceptor), + connect.WithInterceptors(logger_interceptor.NewInterceptor(logger), interceptor), )) srv := startHTTPServer(t, mux) diff --git a/backend/pkg/sqlconnect/sql-connector.go b/backend/pkg/sqlconnect/sql-connector.go index 71040049b..d9c6f3689 100644 --- a/backend/pkg/sqlconnect/sql-connector.go +++ b/backend/pkg/sqlconnect/sql-connector.go @@ -337,7 +337,11 @@ func getGeneralDbConnectConfigFromPg(config *mgmtv1alpha1.ConnectionConfig_PgCon case *mgmtv1alpha1.PostgresConnectionConfig_Url: u, err := url.Parse(cc.Url) if err != nil { - return nil, err + var urlErr *url.Error + if errors.As(err, &urlErr) { + return nil, fmt.Errorf("unable to parse postgres url [%s]: %w", urlErr.Op, urlErr.Err) + } + return nil, fmt.Errorf("unable to parse postgres url: %w", err) } user := u.User.Username() @@ -352,7 +356,7 @@ func getGeneralDbConnectConfigFromPg(config *mgmtv1alpha1.ConnectionConfig_PgCon if portStr != "" { port, err = strconv.ParseInt(portStr, 10, 32) if err != nil { - return nil, fmt.Errorf("invalid port: %v", err) + return nil, fmt.Errorf("invalid port: %w", err) } } else { // default to standard postgres port 5432 if port not provided diff --git a/backend/services/mgmt/v1alpha1/connection-service/connection.go b/backend/services/mgmt/v1alpha1/connection-service/connection.go index 65ce75e99..1cb37f33b 100644 --- a/backend/services/mgmt/v1alpha1/connection-service/connection.go +++ b/backend/services/mgmt/v1alpha1/connection-service/connection.go @@ -37,7 +37,6 @@ func (s *Service) CheckConnectionConfig( schemaTablePrivsMap := make(map[string][]string) conn, err := s.sqlConnector.NewPgPoolFromConnectionConfig(req.Msg.GetConnectionConfig().GetPgConfig(), &connectionTimeout, logger) - if err != nil { return nil, err } @@ -46,14 +45,10 @@ func (s *Service) CheckConnectionConfig( if err != nil { return nil, err } - defer conn.Close() cctx, cancel := context.WithCancel(ctx) defer cancel() - if err != nil { - return nil, err - } switch config := req.Msg.ConnectionConfig.GetPgConfig().ConnectionConfig.(type) { case *mgmtv1alpha1.PostgresConnectionConfig_Connection: @@ -61,7 +56,11 @@ func (s *Service) CheckConnectionConfig( case *mgmtv1alpha1.PostgresConnectionConfig_Url: u, err := url.Parse(config.Url) if err != nil { - return nil, err + var urlErr *url.Error + if errors.As(err, &urlErr) { + return nil, fmt.Errorf("unable to parse postgres url [%s]: %w", urlErr.Op, urlErr.Err) + } + return nil, fmt.Errorf("unable to parse postgres url: %w", err) } role = u.User.Username() } @@ -114,9 +113,6 @@ func (s *Service) CheckConnectionConfig( cctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() - if err != nil { - return nil, err - } err = db.PingContext(cctx) if err != nil { diff --git a/docs/docs/transformers/system.mdx b/docs/docs/transformers/system.md similarity index 97% rename from docs/docs/transformers/system.mdx rename to docs/docs/transformers/system.md index 8b062b563..e9a8f0c9e 100644 --- a/docs/docs/transformers/system.mdx +++ b/docs/docs/transformers/system.md @@ -709,16 +709,16 @@ The transform float transformer can anonymize an existing float64 value or gener For example: `32.2432` -The params `randomizationRangeMin` and `randomizationRangeMax` set an upper and lower bound around the value that you want to anonymize. For example, if the input value is 10, and you set the `randomizationRangeMin` value to 5, then the maximum range will be 5. And if you set the `randomizationRangeMax` to 5, then the maximum range will be 15 ( 10 + 5 = 15). +The params `randomizationRangeMin` and `randomizationRangeMax` set an upper and lower bound around the value that you want to anonymize relative to the input. For example, if the input value is 10, and you set the `randomizationRangeMin` value to 5, then the minimum will be 5. And if you set the `randomizationRangeMax` to 5, then the maximum will be 15 ( 10 + 5 = 15). **Configurations** Depending on your validations, you may want to configure the output float. The random float transformer has the following configurations: -| Name | Description | Default | Example Input | Example Output | -| ----------------------- | ---------------------------------------------------- | ------- | ------------- | -------------- | -| Randomization Range Min | Sets the lower bound of a range for the input value. | 20.00 | 50.23 | 43.30 | -| Randomization Range Max | Sets the upper bound of a range for the input value. | 50.00 | 20.00 | 69.523 | +| Name | Description | Default | +| ------------------ | ---------------------------------------------------- | ------- | +| Relative Range Min | Sets the lower bound of a range for the input value. | 20.00 | +| Relative Range Max | Sets the upper bound of a range for the input value. | 50.00 | **Examples** @@ -790,26 +790,24 @@ The random integer transformer generates a random integer. For example: `6782` -By default, the random int transformer generates an integer of 4 digits long. - **Configurations** Depending on your validations, you may want to configure the output integer. The random integer transformer has the following configurations: -| Name | Description | Default | Example Input | Example Output | -| ----------------------- | ---------------------------------------------------- | ------- | ------------- | -------------- | -| Randomization Range Min | Sets the lower bound of a range for the input value. | 20 | 50 | 43 | -| Randomization Range Max | Sets the upper bound of a range for the input value. | 50 | 20 | 69 | +| Name | Description | Default | +| ------------------ | ------------------------------------------------------------ | ------- | +| Relative Range Min | Sets the lower bound of a range relative to the input value. | 20 | +| Relative Range Max | Sets the upper bound of a range relative to the input value. | 50 | **Examples** There are several ways you can mix-and-match configurations to get different potential random int64 values. Here are some possible combinations: -| Randomization Range Min | Randomization Range Max | Example Input | Example Output | -| ----------------------- | ----------------------- | ------------- | -------------- | -| 5 | 50 | 10 | 43 | -| 10 | 70 | 23 | 91 | -| 20 | 90 | 30 | 27 | +| Relative Range Min | Relative Range Max | Example Input | Example Output | +| ------------------ | ------------------ | ------------- | -------------- | +| 5 | 5 | 10 | 15 | +| 10 | 70 | 23 | 91 | +| 20 | 90 | 30 | 27 | ### Transform Last Name\{#transform-last-name} diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/transformer/UserDefinedTransformerForms/UserDefinedGenerateFloat64Form.tsx b/frontend/apps/web/app/(mgmt)/[account]/new/transformer/UserDefinedTransformerForms/UserDefinedGenerateFloat64Form.tsx index 687c1b65f..f667ebe01 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/transformer/UserDefinedTransformerForms/UserDefinedGenerateFloat64Form.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/new/transformer/UserDefinedTransformerForms/UserDefinedGenerateFloat64Form.tsx @@ -101,7 +101,7 @@ export default function UserDefinedGenerateFloat64Form( render={({ field }) => (
- Maxiumum Value + Maximum Value Sets a maximum range for generated float64 value. This can be negative as well. diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/transformer/UserDefinedTransformerForms/UserDefinedGenerateInt64Form.tsx b/frontend/apps/web/app/(mgmt)/[account]/new/transformer/UserDefinedTransformerForms/UserDefinedGenerateInt64Form.tsx index 8a97e8039..ea938356f 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/transformer/UserDefinedTransformerForms/UserDefinedGenerateInt64Form.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/new/transformer/UserDefinedTransformerForms/UserDefinedGenerateInt64Form.tsx @@ -101,7 +101,7 @@ export default function UserDefinedGenerateInt64Form( render={({ field }) => (
- Maxiumum Value + Maximum Value Sets a maximum range for generated int64 value. This can be negative as well. diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/transformer/UserDefinedTransformerForms/UserDefinedTransformFloat64Form.tsx b/frontend/apps/web/app/(mgmt)/[account]/new/transformer/UserDefinedTransformerForms/UserDefinedTransformFloat64Form.tsx index aacd075a8..df12119da 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/transformer/UserDefinedTransformerForms/UserDefinedTransformFloat64Form.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/new/transformer/UserDefinedTransformerForms/UserDefinedTransformFloat64Form.tsx @@ -35,12 +35,12 @@ export default function UserDefinedTransformFloat64Form( render={({ field }) => (
- Minimum Range Value + Relative Minimum Range Value - Sets a minium lower range value. This will create an lowerbound - around the source input value. For example, if the input value - is 10, and you set this value to 5, then the minimum range will - be 5 (10-5 = 5). + Sets a relative minium lower range value. This will create a + lowerbound around the source input value. For example, if the + input value is 10, and you set this value to 5, then the minimum + range will be 5 (10-5 = 5).
@@ -71,12 +71,12 @@ export default function UserDefinedTransformFloat64Form( render={({ field }) => (
- Maxiumum Range Value + Relative Maximum Range Value - Sets a maximum upper range value. This will create an upperbound - around the source input value. For example, if the input value - is 10, and you set this value to 5, then the maximum range will - be 15 ( 10 + 5 = 15). + Sets a relative maximum upper range value. This will create an + upperbound around the source input value. For example, if the + input value is 10, and you set this value to 5, then the maximum + range will be 15 ( 10 + 5 = 15).
diff --git a/frontend/apps/web/app/(mgmt)/[account]/new/transformer/UserDefinedTransformerForms/UserDefinedTransformInt64Form.tsx b/frontend/apps/web/app/(mgmt)/[account]/new/transformer/UserDefinedTransformerForms/UserDefinedTransformInt64Form.tsx index 07ef5428f..b95fbd65e 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/new/transformer/UserDefinedTransformerForms/UserDefinedTransformInt64Form.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/new/transformer/UserDefinedTransformerForms/UserDefinedTransformInt64Form.tsx @@ -35,12 +35,12 @@ export default function UserDefinedTransformInt64Form( render={({ field }) => (
- Minimum Range Value + Relative Minimum Range Value - Sets a minium lower range value. This will create an lowerbound - around the source input value. For example, if the input value - is 10, and you set this value to 5, then the maximum range will - be 5. + Sets a relative minium lower range value. This will create a + lowerbound around the source input value. For example, if the + input value is 10, and you set this value to 5, then the maximum + range will be 5.
@@ -71,12 +71,12 @@ export default function UserDefinedTransformInt64Form( render={({ field }) => (
- Maxiumum Range Value + Relative Maximum Range Value - Sets a maximum upper range value. This will create an upperbound - around the source input value. For example, if the input value - is 10, and you set this value to 5, then the maximum range will - be 15. + Sets a relative maximum upper range value. This will create an + upperbound around the source input value. For example, if the + input value is 10, and you set this value to 5, then the maximum + range will be 15.
diff --git a/frontend/apps/web/app/(mgmt)/[account]/transformers/Sheetforms/GenerateFloat64Form.tsx b/frontend/apps/web/app/(mgmt)/[account]/transformers/Sheetforms/GenerateFloat64Form.tsx index 4a0395c32..085a3d0e2 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/transformers/Sheetforms/GenerateFloat64Form.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/transformers/Sheetforms/GenerateFloat64Form.tsx @@ -93,7 +93,7 @@ export default function GenerateFloat64Form(props: Props): ReactElement { render={({ field }) => (
- Maxiumum Value + Maximum Value Sets a maximum range for generated float64 value. This can be negative as well. diff --git a/frontend/apps/web/app/(mgmt)/[account]/transformers/Sheetforms/GenerateInt64Form.tsx b/frontend/apps/web/app/(mgmt)/[account]/transformers/Sheetforms/GenerateInt64Form.tsx index 710785612..c1233da4a 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/transformers/Sheetforms/GenerateInt64Form.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/transformers/Sheetforms/GenerateInt64Form.tsx @@ -95,7 +95,7 @@ export default function GenerateInt64Form(props: Props): ReactElement { render={({ field }) => (
- Maxiumum Value + Maximum Value Sets a maximum range for generated int64 value. This can be negative as well. diff --git a/frontend/apps/web/app/(mgmt)/[account]/transformers/Sheetforms/TransformFloat64Form.tsx b/frontend/apps/web/app/(mgmt)/[account]/transformers/Sheetforms/TransformFloat64Form.tsx index 6bfcd2392..244f3081a 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/transformers/Sheetforms/TransformFloat64Form.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/transformers/Sheetforms/TransformFloat64Form.tsx @@ -40,9 +40,9 @@ export default function TransformFloat64Form(props: Props): ReactElement { render={({ field }) => (
- Minimum Value + Relative Minimum Value - Sets a minium lower range value. This will create an + Sets a relative minium lower range value. This will create an lowerbound around the source input value. For example, if the input value is 10, and you set this value to 5, then the maximum range will be 5. @@ -72,9 +72,9 @@ export default function TransformFloat64Form(props: Props): ReactElement { render={({ field }) => (
- Maxiumum Range Value + Relative Maximum Range Value - Sets a maximum upper range value. This will create an + Sets a relative maximum upper range value. This will create an upperbound around the source input value. For example, if the input value is 10, and you set this value to 5, then the maximum range will be 15. diff --git a/frontend/apps/web/app/(mgmt)/[account]/transformers/Sheetforms/TransformInt64Form.tsx b/frontend/apps/web/app/(mgmt)/[account]/transformers/Sheetforms/TransformInt64Form.tsx index b3705a973..43b58dc74 100644 --- a/frontend/apps/web/app/(mgmt)/[account]/transformers/Sheetforms/TransformInt64Form.tsx +++ b/frontend/apps/web/app/(mgmt)/[account]/transformers/Sheetforms/TransformInt64Form.tsx @@ -41,9 +41,9 @@ export default function TransformInt64Form(props: Props): ReactElement { render={({ field }) => (
- Minimum Range Value + Relative Minimum Range Value - Sets a minium lower range value. This will create an + Sets a relative minium lower range value. This will create an lowerbound around the source input value. For example, if the input value is 10, and you set this value to 5, then the maximum range will be 5. @@ -75,9 +75,9 @@ export default function TransformInt64Form(props: Props): ReactElement { render={({ field }) => (
- Maxiumum Range Value + Relative Maximum Range Value - Sets a maximum upper range value. This will create an + Sets a relative maximum upper range value. This will create an upperbound around the source input value. For example, if the input value is 10, and you set this value to 5, then the maximum range will be 15. diff --git a/worker/internal/benthos/transformers/transform_float.go b/worker/internal/benthos/transformers/transform_float.go index d85d3cb6c..bb7ef3796 100644 --- a/worker/internal/benthos/transformers/transform_float.go +++ b/worker/internal/benthos/transformers/transform_float.go @@ -19,10 +19,6 @@ func init() { return nil, err } - var value float64 - if valuePtr != nil { - value = *valuePtr - } rMin, err := args.GetFloat64("randomization_range_min") if err != nil { return nil, err @@ -33,7 +29,7 @@ func init() { return nil, err } return func() (any, error) { - res, err := TransformFloat(value, rMin, rMax) + res, err := transformFloat(valuePtr, rMin, rMax) if err != nil { return nil, fmt.Errorf("unable to run transform_float64: %w", err) } @@ -46,25 +42,17 @@ func init() { } } -func TransformFloat(value, rMin, rMax float64) (*float64, error) { - if value == 0 { +func transformFloat(value *float64, rMin, rMax float64) (*float64, error) { + if value == nil { return nil, nil } - // require that the value is in the randomization range so that we can transform it otherwise, should use the generate_int transformer - - if !transformer_utils.IsFloat64InRandomizationRange(value, rMin, rMax) { - zeroVal := float64(0) - return &zeroVal, fmt.Errorf("the value is not the provided range") - } - - minRange := value - rMin - maxRange := value + rMax + minRange := *value - rMin + maxRange := *value + rMax val, err := transformer_utils.GenerateRandomFloat64WithInclusiveBounds(minRange, maxRange) if err != nil { return nil, fmt.Errorf("unable to generate a random float64 with inclusive bounds with length [%f:%f]: %w", minRange, maxRange, err) } - return &val, nil } diff --git a/worker/internal/benthos/transformers/transform_float_test.go b/worker/internal/benthos/transformers/transform_float_test.go index a5aee24d4..510b127b2 100644 --- a/worker/internal/benthos/transformers/transform_float_test.go +++ b/worker/internal/benthos/transformers/transform_float_test.go @@ -5,42 +5,32 @@ import ( "testing" "github.com/benthosdev/benthos/v4/public/bloblang" - transformer_utils "github.com/nucleuscloud/neosync/worker/internal/benthos/transformers/utils" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func Test_TransformFloat64ErrorNotInRange(t *testing.T) { - val := float64(27.1) - rMin := float64(22.9) - rMax := float64(25.333) - - res := transformer_utils.IsFloat64InRandomizationRange(val, rMin, rMax) - assert.Equal(t, false, res, "The value should not be in the range") -} - func Test_TransformFloat64InRange(t *testing.T) { val := float64(27.2323) - rMin := float64(22.12) - rMax := float64(29.9823) + rMin := float64(5) + rMax := float64(5) - res, err := TransformFloat(val, rMin, rMax) - assert.NoError(t, err) + res, err := transformFloat(&val, rMin, rMax) + require.NoError(t, err) - assert.GreaterOrEqual(t, *res, val-rMin, "The result should be greater than the min") - assert.LessOrEqual(t, *res, val+rMax, "The result should be less than the max") + require.GreaterOrEqual(t, *res, val-rMin, "The result should be greater than the min") + require.LessOrEqual(t, *res, val+rMax, "The result should be less than the max") } -func Test_TransformFloat64PhoneTransformerWithNilValue(t *testing.T) { +func Test_TransformFloat64_Benthos(t *testing.T) { val := float64(27.35) rMin := float64(22.24) rMax := float64(29.928) mapping := fmt.Sprintf(`root = transform_float64(value:%f, randomization_range_min:%f,randomization_range_max: %f)`, val, rMin, rMax) ex, err := bloblang.Parse(mapping) - assert.NoError(t, err, "failed to parse the email transformer") + require.NoError(t, err, "failed to parse the email transformer") res, err := ex.Query(nil) - assert.NoError(t, err) + require.NoError(t, err) resInt, ok := res.(*float64) if !ok { @@ -49,9 +39,9 @@ func Test_TransformFloat64PhoneTransformerWithNilValue(t *testing.T) { } if resInt != nil { - assert.GreaterOrEqual(t, *resInt, val-rMin, "The result should be greater than the min") - assert.LessOrEqual(t, *resInt, val+rMax, "The result should be less than the max") + require.GreaterOrEqual(t, *resInt, val-rMin, "The result should be greater than the min") + require.LessOrEqual(t, *resInt, val+rMax, "The result should be less than the max") } else { - assert.Error(t, err, "Expected the pointer to resolve to an float64") + require.Error(t, err, "Expected the pointer to resolve to an float64") } } diff --git a/worker/internal/benthos/transformers/transform_int64.go b/worker/internal/benthos/transformers/transform_int64.go index 660fc0c0c..7aaebf02e 100644 --- a/worker/internal/benthos/transformers/transform_int64.go +++ b/worker/internal/benthos/transformers/transform_int64.go @@ -19,11 +19,6 @@ func init() { return nil, err } - var value int64 - if valuePtr != nil { - value = *valuePtr - } - rMin, err := args.GetInt64("randomization_range_min") if err != nil { return nil, err @@ -34,7 +29,7 @@ func init() { return nil, err } return func() (any, error) { - res, err := TransformInt(value, rMin, rMax) + res, err := transformInt(valuePtr, rMin, rMax) if err != nil { return nil, fmt.Errorf("unable to run transform_int64: %w", err) } @@ -47,21 +42,13 @@ func init() { } } -func TransformInt(value, rMin, rMax int64) (*int64, error) { - if value == 0 { +func transformInt(value *int64, rMin, rMax int64) (*int64, error) { + if value == nil { return nil, nil } - // require that the value is in the randomization range so that we can transform it - // otherwise, should use the generate_int transformer - - if !transformer_utils.IsIntInRandomizationRange(value, rMin, rMax) { - zeroVal := int64(0) - return &zeroVal, fmt.Errorf("the value is not the provided range") - } - - minRange := value - rMin - maxRange := value + rMax + minRange := *value - rMin + maxRange := *value + rMax val, err := transformer_utils.GenerateRandomInt64InValueRange(minRange, maxRange) if err != nil { diff --git a/worker/internal/benthos/transformers/transform_int64_test.go b/worker/internal/benthos/transformers/transform_int64_test.go index c20932e98..8f9d709c7 100644 --- a/worker/internal/benthos/transformers/transform_int64_test.go +++ b/worker/internal/benthos/transformers/transform_int64_test.go @@ -5,42 +5,32 @@ import ( "testing" "github.com/benthosdev/benthos/v4/public/bloblang" - transformer_utils "github.com/nucleuscloud/neosync/worker/internal/benthos/transformers/utils" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func Test_TransformIntErrorNotInRange(t *testing.T) { - val := int64(27) - rMin := int64(22) - rMax := int64(25) - - res := transformer_utils.IsIntInRandomizationRange(val, rMin, rMax) - assert.Equal(t, false, res, "The value should not be in the range") -} - func Test_TransformIntInRange(t *testing.T) { val := int64(27) - rMin := int64(22) - rMax := int64(29) + rMin := int64(5) + rMax := int64(5) - res, err := TransformInt(val, rMin, rMax) - assert.NoError(t, err) + res, err := transformInt(&val, rMin, rMax) + require.NoError(t, err) - assert.GreaterOrEqual(t, *res, val-rMin, "The result should be greater than the min") - assert.LessOrEqual(t, *res, val+rMax, "The result should be less than the max") + require.GreaterOrEqual(t, *res, val-rMin, "The result should be greater than the min") + require.LessOrEqual(t, *res, val+rMax, "The result should be less than the max") } -func Test_TransformIntPhoneTransformerWithNilValue(t *testing.T) { +func Test_TransformInt64_Benthos(t *testing.T) { val := int64(27) - rMin := int64(22) - rMax := int64(29) + rMin := int64(5) + rMax := int64(5) mapping := fmt.Sprintf(`root = transform_int64(value:%d, randomization_range_min:%d,randomization_range_max: %d)`, val, rMin, rMax) ex, err := bloblang.Parse(mapping) - assert.NoError(t, err, "failed to parse the email transformer") + require.NoError(t, err, "failed to parse the email transformer") res, err := ex.Query(nil) - assert.NoError(t, err) + require.NoError(t, err) resInt, ok := res.(*int64) if !ok { @@ -49,9 +39,9 @@ func Test_TransformIntPhoneTransformerWithNilValue(t *testing.T) { } if resInt != nil { - assert.GreaterOrEqual(t, *resInt, val-rMin, "The result should be greater than the min") - assert.LessOrEqual(t, *resInt, val+rMax, "The result should be less than the max") + require.GreaterOrEqual(t, *resInt, val-rMin, "The result should be greater than the min") + require.LessOrEqual(t, *resInt, val+rMax, "The result should be less than the max") } else { - assert.Error(t, err, "Expected the pointer to resolve to an int64") + require.Error(t, err, "Expected the pointer to resolve to an int64") } } diff --git a/worker/internal/benthos/transformers/utils/float_utils.go b/worker/internal/benthos/transformers/utils/float_utils.go index 369df1715..2f814a160 100644 --- a/worker/internal/benthos/transformers/utils/float_utils.go +++ b/worker/internal/benthos/transformers/utils/float_utils.go @@ -25,7 +25,6 @@ func GenerateRandomFloat64WithInclusiveBounds(min, max float64) (float64, error) // Scale and shift the value to the range returnValue := min + randValue*(max-min) - return returnValue, nil } @@ -67,18 +66,6 @@ func IsNegativeFloat64(val float64) bool { } } -func IsFloat64InRandomizationRange(value, rMin, rMax float64) bool { - if rMin > rMax { - rMin, rMax = rMax, rMin - } - - if rMin == rMax { - return value == rMin - } - - return value >= rMin && value <= rMax -} - type FloatLength struct { DigitsBeforeDecimalLength int64 DigitsAfterDecimalLength int64 diff --git a/worker/internal/benthos/transformers/utils/float_utils_test.go b/worker/internal/benthos/transformers/utils/float_utils_test.go index 1958354ea..b17db66d9 100644 --- a/worker/internal/benthos/transformers/utils/float_utils_test.go +++ b/worker/internal/benthos/transformers/utils/float_utils_test.go @@ -92,24 +92,6 @@ func Test_IsNegativeFloatFalse(t *testing.T) { assert.False(t, val, "The value should be positive") } -func Test_IsFloat64InRandomizationRangeTrue(t *testing.T) { - val := float64(27) - rMin := float64(22) - rMax := float64(29) - - res := IsFloat64InRandomizationRange(val, rMin, rMax) - assert.Equal(t, true, res, "The value should be in the range") -} - -func Test_IsFloat64InRandomizationRangeFalse(t *testing.T) { - val := float64(27) - rMin := float64(22) - rMax := float64(25) - - res := IsFloat64InRandomizationRange(val, rMin, rMax) - assert.Equal(t, false, res, "The value should not be in the range") -} - func Test_GetFloatLength(t *testing.T) { val := float64(3.14) res := GetFloatLength(val) diff --git a/worker/internal/benthos/transformers/utils/integer_utils.go b/worker/internal/benthos/transformers/utils/integer_utils.go index 4d7f1ae44..5d09d7174 100644 --- a/worker/internal/benthos/transformers/utils/integer_utils.go +++ b/worker/internal/benthos/transformers/utils/integer_utils.go @@ -92,18 +92,6 @@ func AbsInt[T int | int64 | int32 | uint | uint32 | uint64](n T) T { return n } -func IsIntInRandomizationRange[T int | int64 | int32 | uint | uint32 | uint64](value, rMin, rMax T) bool { - if rMin > rMax { - rMin, rMax = rMax, rMin - } - - if rMin == rMax { - return value == rMin - } - - return value >= rMin && value <= rMax -} - func MinInt[T int | int64 | int32 | uint | uint32 | uint64](a, b T) T { if a < b { return a diff --git a/worker/internal/benthos/transformers/utils/integer_utils_test.go b/worker/internal/benthos/transformers/utils/integer_utils_test.go index 33d58b805..34b15c828 100644 --- a/worker/internal/benthos/transformers/utils/integer_utils_test.go +++ b/worker/internal/benthos/transformers/utils/integer_utils_test.go @@ -133,24 +133,6 @@ func Test_IsNegativeIntFalse(t *testing.T) { assert.False(t, val, "The value should be positive") } -func Test_IsValueInRandomizationRangeTrue(t *testing.T) { - val := int64(27) - rMin := int64(22) - rMax := int64(29) - - res := IsIntInRandomizationRange(val, rMin, rMax) - assert.Equal(t, true, res, "The value should be in the range") -} - -func Test_IsValueInRandomizationRangeFalse(t *testing.T) { - val := int64(27) - rMin := int64(22) - rMax := int64(25) - - res := IsIntInRandomizationRange(val, rMin, rMax) - assert.Equal(t, false, res, "The value should not be in the range") -} - func Test_MinInt(t *testing.T) { assert.Equal(t, 1, MinInt(1, 2)) assert.Equal(t, 1, MinInt(2, 1))