From 4c46724ec7c2a86a334134e6c06e3c836010cbf9 Mon Sep 17 00:00:00 2001 From: Yevgeniy Firsov Date: Fri, 12 May 2023 12:09:52 -0700 Subject: [PATCH 1/9] fix: Support new productions clusters --- login/login.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/login/login.go b/login/login.go index f04d4b3..5220177 100644 --- a/login/login.go +++ b/login/login.go @@ -48,6 +48,12 @@ type instance struct { var ( ErrStateMismatched = fmt.Errorf("state is not matched") ErrInstanceNotFound = fmt.Errorf("instance not found") + + defaultInstance = instance{ + clientID: "GS8PrHA1aYblUR73yitqomc40ZYZ81jF", + authHost: "https://auth.tigrisdata.cloud/", + audience: "https://tigris-api-prod", + } ) var ( @@ -295,7 +301,11 @@ func CmdLow(_ context.Context, host string) error { inst, ok := instances[host] if !ok { - return util.Error(fmt.Errorf("%w: %s", ErrInstanceNotFound, host), "Instance config not found") + if !strings.HasSuffix(host, ".tigrisdata.cloud") { + return util.Error(fmt.Errorf("%w: %s", ErrInstanceNotFound, host), "Instance config not found") + } + + inst = defaultInstance } p, err := oidc.NewProvider(context.Background(), inst.authHost) From c7a042d551615d97318d3e4ec0463c4c02dded20 Mon Sep 17 00:00:00 2001 From: Yevgeniy Firsov Date: Fri, 12 May 2023 12:25:04 -0700 Subject: [PATCH 2/9] fix: Upgrade dependencies --- go.mod | 10 +++++----- go.sum | 22 +++++++++++----------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 2ec2453..2150e75 100644 --- a/go.mod +++ b/go.mod @@ -19,9 +19,9 @@ require ( github.com/spf13/cobra v1.7.0 github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.2 - github.com/tigrisdata/tigris-client-go v1.0.0-beta.37 + github.com/tigrisdata/tigris-client-go v1.0.0 golang.org/x/net v0.10.0 - golang.org/x/oauth2 v0.7.0 + golang.org/x/oauth2 v0.8.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -36,7 +36,7 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/deepmap/oapi-codegen v1.12.4 // indirect - github.com/docker/distribution v2.8.1+incompatible // indirect + github.com/docker/distribution v2.8.2+incompatible // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-git/gcfg v1.5.0 // indirect @@ -79,12 +79,12 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect - golang.org/x/crypto v0.8.0 // indirect + golang.org/x/crypto v0.9.0 // indirect golang.org/x/mod v0.10.0 // indirect golang.org/x/sys v0.8.0 // indirect golang.org/x/term v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect - golang.org/x/tools v0.8.0 // indirect + golang.org/x/tools v0.9.1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/grpc v1.55.0 // indirect diff --git a/go.sum b/go.sum index 7e1551e..b99f7ad 100644 --- a/go.sum +++ b/go.sum @@ -92,8 +92,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deepmap/oapi-codegen v1.12.4 h1:pPmn6qI9MuOtCz82WY2Xaw46EQjgvxednXXrP7g5Q2s= github.com/deepmap/oapi-codegen v1.12.4/go.mod h1:3lgHGMu6myQ2vqbbTXH2H1o4eXFTGnFiDaOaKKl5yas= -github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= -github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v23.0.6+incompatible h1:aBD4np894vatVX99UTx/GyOUOK4uEcROwA3+bQhEcoU= github.com/docker/docker v23.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= @@ -355,8 +355,8 @@ github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= -github.com/tigrisdata/tigris-client-go v1.0.0-beta.37 h1:YFwjiEiexMDHTudzdrfGk9I7TAIj/mamsp/XbfNFCrE= -github.com/tigrisdata/tigris-client-go v1.0.0-beta.37/go.mod h1:2n6TQUdoTbzuTtakHT/ZNuK5X+I/i57BqqCcYAzG7y4= +github.com/tigrisdata/tigris-client-go v1.0.0 h1:07Qw8Tm0qL15WiadP0hp4iBiRzfNSJ+GH4/ozO0nNs0= +github.com/tigrisdata/tigris-client-go v1.0.0/go.mod h1:2n6TQUdoTbzuTtakHT/ZNuK5X+I/i57BqqCcYAzG7y4= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -395,8 +395,8 @@ golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= -golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -490,8 +490,8 @@ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -504,8 +504,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -649,8 +649,8 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= -golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From c70cd44c3db8baa9e256b919f0245b0a57783c85 Mon Sep 17 00:00:00 2001 From: Yevgeniy Firsov Date: Fri, 12 May 2023 21:53:01 -0700 Subject: [PATCH 3/9] fix: Fix float numbers for search on import This change detects if document contains array of objects in the schema, and if it is the case replaces all the numbers to fractional value, in order to guarantee that search infer their schema as float. It inserts this record with replaced numbers for search to persists proper schema and deletes the record immediately. It proceeds import process as usual after that --- cmd/import.go | 64 +++++++++++++++++++++++++++++++++++++++++++++ schema/inference.go | 19 ++++++++++++++ util/util.go | 2 +- 3 files changed, 84 insertions(+), 1 deletion(-) diff --git a/cmd/import.go b/cmd/import.go index b8d2d5c..b858d88 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -17,6 +17,7 @@ package cmd import ( "context" "encoding/json" + "errors" "fmt" "unsafe" @@ -54,6 +55,10 @@ var ( ErrCollectionShouldExist = fmt.Errorf("collection should exist to import CSV with no field names") ErrNoAppend = fmt.Errorf( "collection exists. use --append if you need to add documents to existing collection") + + ErrNoRecordsExpected = fmt.Errorf("no records expected in the collection after fixing numbers") + + FirstRecord = true ) func evolveSchema(ctx context.Context, db string, coll string, docs []json.RawMessage) error { @@ -74,7 +79,66 @@ func evolveSchema(ctx context.Context, db string, coll string, docs []json.RawMe return util.Error(err, "create or update collection") } +func fixNumbers(ctx context.Context, coll string, docs []json.RawMessage) { + err := schema.Infer(&sch, coll, docs[0:1], PrimaryKey, AutoGenerate, 1) + util.Fatal(err, "infer schema") + + if schema.HasArrayOfObjects { + b, err := json.Marshal(schema.DummyRecord) + util.Fatal(err, "marshal number fixer record") + + _, err = client.GetDB().Insert(ctx, coll, []driver.Document{b}) + util.Fatal(err, "insert") + + _, err = client.GetDB().Delete(ctx, coll, driver.Filter("{}")) + util.Fatal(err, "delete") + } +} + +func guaranteeFloatsInFirstRecord(ctx context.Context, coll string, docs []json.RawMessage) { + if !FirstRecord { + return + } + + cnt, err := client.GetDB().Count(ctx, coll, driver.Filter("{}")) + + var ep *driver.Error + + if err != nil { + if errors.As(err, &ep) && ep.Code == api.Code_NOT_FOUND { + log.Debug().Msg("collection doesn't exits, skip fixing numbers") + return + } + + util.Fatal(err, "get count") + } + + if cnt == 0 { + schema.DetectArrayOfObjects = true + schema.DetectIntegers = false + schema.ReplaceNumber = true + + fixNumbers(ctx, coll, docs) + + schema.DetectArrayOfObjects = false + schema.DetectIntegers = true + schema.ReplaceNumber = false + + cnt, err := client.GetDB().Count(ctx, coll, driver.Filter("{}")) + util.Fatal(err, "get count after") + + if cnt != 0 { + util.Fatal(ErrNoRecordsExpected, "checking number of records after") + } + } + + FirstRecord = false +} + func insertWithInference(ctx context.Context, coll string, docs []json.RawMessage) error { + // FIXME: This is temporary fix, should moved to server ASAP + guaranteeFloatsInFirstRecord(ctx, coll, docs) + ptr := unsafe.Pointer(&docs) _, err := client.GetDB().Insert(ctx, coll, *(*[]driver.Document)(ptr)) diff --git a/schema/inference.go b/schema/inference.go index d7f564e..2793e6d 100644 --- a/schema/inference.go +++ b/schema/inference.go @@ -54,6 +54,11 @@ var ( ErrExpectedString = fmt.Errorf("expected string type") ErrExpectedNumber = fmt.Errorf("expected json.Number") ErrUnsupportedType = fmt.Errorf("unsupported type") + + HasArrayOfObjects bool + DummyRecord map[string]any + DetectArrayOfObjects bool + ReplaceNumber bool ) func newInompatibleSchemaError(name, oldType, oldFormat, newType, newFormat string) error { @@ -249,6 +254,12 @@ func traverseArray(name string, existingField *schema.Field, newField *schema.Fi newField.Items.Format = nf if t == typeObject { + if DetectArrayOfObjects { + log.Debug().Msg("detected array of objects") + + HasArrayOfObjects = true + } + values, _ := reflect.ValueOf(v).Index(i).Interface().(map[string]any) if err = traverseObject(name, newField.Items, newField.Items, values); err != nil { return err @@ -335,6 +346,10 @@ func traverseFields(sch map[string]*schema.Field, fields map[string]any, autoGen continue } + if t == typeNumber && ReplaceNumber { + fields[name] = 1.1 + } + setAutoGenerate(autoGen, name, f) sch[name] = f @@ -373,6 +388,10 @@ func docToSchema(sch *schema.Schema, name string, data []byte, pk []string, auto sch.PrimaryKey = []string{"id"} } + if ReplaceNumber { + DummyRecord = m + } + return nil } diff --git a/util/util.go b/util/util.go index db70583..e1cc9eb 100644 --- a/util/util.go +++ b/util/util.go @@ -114,7 +114,7 @@ func Infof(format string, args ...interface{}) { } func Error(err error, msg string, args ...interface{}) error { - log.Err(err).CallerSkipFrame(3).Msgf(msg, args...) + log.Err(err).CallerSkipFrame(2).Msgf(msg, args...) if err == nil { return nil From 1f1d50f766dba9aa8261dfadf1f624c237bf326a Mon Sep 17 00:00:00 2001 From: Yevgeniy Firsov Date: Sat, 13 May 2023 10:14:05 -0700 Subject: [PATCH 4/9] fix: Fix numbers --- schema/inference.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema/inference.go b/schema/inference.go index 2793e6d..cb4971e 100644 --- a/schema/inference.go +++ b/schema/inference.go @@ -346,7 +346,7 @@ func traverseFields(sch map[string]*schema.Field, fields map[string]any, autoGen continue } - if t == typeNumber && ReplaceNumber { + if sch != nil && sch[name] != nil && sch[name].Type == typeNumber && ReplaceNumber { fields[name] = 1.1 } From 7560dfd9e98e84cd01d219bdf65902692517c9a9 Mon Sep 17 00:00:00 2001 From: Yevgeniy Firsov Date: Sat, 13 May 2023 13:03:27 -0700 Subject: [PATCH 5/9] fix: Do not extend integer to float if it's set by existing schema Same for UUID, Times, byte arrays. Obey the schema. --- cmd/import.go | 2 -- schema/inference.go | 29 ++++++++++++++++------------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/cmd/import.go b/cmd/import.go index b858d88..ed47368 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -115,13 +115,11 @@ func guaranteeFloatsInFirstRecord(ctx context.Context, coll string, docs []json. if cnt == 0 { schema.DetectArrayOfObjects = true - schema.DetectIntegers = false schema.ReplaceNumber = true fixNumbers(ctx, coll, docs) schema.DetectArrayOfObjects = false - schema.DetectIntegers = true schema.ReplaceNumber = false cnt, err := client.GetDB().Count(ctx, coll, driver.Filter("{}")) diff --git a/schema/inference.go b/schema/inference.go index cb4971e..4803d17 100644 --- a/schema/inference.go +++ b/schema/inference.go @@ -76,15 +76,14 @@ func parseDateTime(s string) bool { return false } -func parseNumber(v any) (string, string, error) { +func parseNumber(v any, existing *schema.Field) (string, string, error) { n, ok := v.(json.Number) if !ok { return "", "", ErrExpectedNumber } - if _, err := n.Int64(); err != nil || !DetectIntegers { - _, err = n.Float64() - if err != nil { + if _, err := n.Int64(); err != nil || (!DetectIntegers && (existing == nil || existing.Type != typeInteger)) { + if _, err = n.Float64(); err != nil { return "", "", err } @@ -94,11 +93,15 @@ func parseNumber(v any) (string, string, error) { return typeInteger, "", nil } -func translateStringType(v interface{}) (string, string, error) { +func needNarrowing(detect bool, existing *schema.Field, format string) bool { + return detect || existing != nil && existing.Format == format +} + +func translateStringType(v any, existing *schema.Field) (string, string, error) { t := reflect.TypeOf(v) if t.PkgPath() == "encoding/json" && t.Name() == "Number" { - return parseNumber(v) + return parseNumber(v, existing) } s, ok := v.(string) @@ -106,15 +109,15 @@ func translateStringType(v interface{}) (string, string, error) { return "", "", ErrExpectedString } - if parseDateTime(s) && DetectTimes { + if parseDateTime(s) && needNarrowing(DetectTimes, existing, formatDateTime) { return typeString, formatDateTime, nil } - if _, err := uuid.Parse(s); err == nil && DetectUUIDs { + if _, err := uuid.Parse(s); err == nil && needNarrowing(DetectUUIDs, existing, formatUUID) { return typeString, formatUUID, nil } - if len(s) != 0 && DetectByteArrays { + if len(s) != 0 && needNarrowing(DetectByteArrays, existing, formatByte) { b := make([]byte, base64.StdEncoding.DecodedLen(len(s))) if _, err := base64.StdEncoding.Decode(b, []byte(s)); err == nil { return typeString, formatByte, nil @@ -124,7 +127,7 @@ func translateStringType(v interface{}) (string, string, error) { return typeString, "", nil } -func translateType(v interface{}) (string, string, error) { +func translateType(v any, existing *schema.Field) (string, string, error) { t := reflect.TypeOf(v) //nolint:golint,exhaustive @@ -134,7 +137,7 @@ func translateType(v interface{}) (string, string, error) { case reflect.Float64: return typeNumber, "", nil case reflect.String: - return translateStringType(v) + return translateStringType(v, existing) case reflect.Slice, reflect.Array: return typeArray, "", nil case reflect.Map: @@ -226,7 +229,7 @@ func traverseObject(name string, existingField *schema.Field, newField *schema.F func traverseArray(name string, existingField *schema.Field, newField *schema.Field, v any) error { for i := 0; i < reflect.ValueOf(v).Len(); i++ { - t, format, err := translateType(reflect.ValueOf(v).Index(i).Interface()) + t, format, err := translateType(reflect.ValueOf(v).Index(i).Interface(), existingField) if err != nil { return err } @@ -330,7 +333,7 @@ func traverseFields(sch map[string]*schema.Field, fields map[string]any, autoGen continue } - t, format, err := translateType(val) + t, format, err := translateType(val, sch[name]) if err != nil { return err } From 921b424b31c27714165764027ba37ec4e9867a27 Mon Sep 17 00:00:00 2001 From: Yevgeniy Firsov Date: Sat, 13 May 2023 13:26:36 -0700 Subject: [PATCH 6/9] fix: Wrap fixing number in a transaction --- cmd/import.go | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/cmd/import.go b/cmd/import.go index ed47368..2b35108 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -87,11 +87,16 @@ func fixNumbers(ctx context.Context, coll string, docs []json.RawMessage) { b, err := json.Marshal(schema.DummyRecord) util.Fatal(err, "marshal number fixer record") - _, err = client.GetDB().Insert(ctx, coll, []driver.Document{b}) - util.Fatal(err, "insert") + err = client.Transact(ctx, config.GetProjectName(), func(ctx context.Context, tx driver.Tx) error { + _, err = tx.Insert(ctx, coll, []driver.Document{b}) + util.Fatal(err, "insert") - _, err = client.GetDB().Delete(ctx, coll, driver.Filter("{}")) - util.Fatal(err, "delete") + _, err = tx.Delete(ctx, coll, driver.Filter("{}")) + util.Fatal(err, "delete") + + return nil + }) + util.Fatal(err, "fix number transaction") } } @@ -121,13 +126,6 @@ func guaranteeFloatsInFirstRecord(ctx context.Context, coll string, docs []json. schema.DetectArrayOfObjects = false schema.ReplaceNumber = false - - cnt, err := client.GetDB().Count(ctx, coll, driver.Filter("{}")) - util.Fatal(err, "get count after") - - if cnt != 0 { - util.Fatal(ErrNoRecordsExpected, "checking number of records after") - } } FirstRecord = false From 644a8250d7170920250b1b5102beb698dfd143f7 Mon Sep 17 00:00:00 2001 From: Ovais Tariq Date: Sat, 13 May 2023 19:23:54 -0700 Subject: [PATCH 7/9] fix: generate init record before importing in collection (#266) --- cmd/import.go | 53 ++++++++++++++------------------- schema/inference.go | 72 +++++++++++++++++++++++++++++++++++---------- 2 files changed, 78 insertions(+), 47 deletions(-) diff --git a/cmd/import.go b/cmd/import.go index 2b35108..a7d7862 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -79,61 +79,52 @@ func evolveSchema(ctx context.Context, db string, coll string, docs []json.RawMe return util.Error(err, "create or update collection") } -func fixNumbers(ctx context.Context, coll string, docs []json.RawMessage) { - err := schema.Infer(&sch, coll, docs[0:1], PrimaryKey, AutoGenerate, 1) - util.Fatal(err, "infer schema") - - if schema.HasArrayOfObjects { - b, err := json.Marshal(schema.DummyRecord) - util.Fatal(err, "marshal number fixer record") - - err = client.Transact(ctx, config.GetProjectName(), func(ctx context.Context, tx driver.Tx) error { - _, err = tx.Insert(ctx, coll, []driver.Document{b}) - util.Fatal(err, "insert") - - _, err = tx.Delete(ctx, coll, driver.Filter("{}")) - util.Fatal(err, "delete") +func writeInitRecord(ctx context.Context, coll string, docs []json.RawMessage) { + initDoc, err := schema.GenerateInitDoc(&sch, docs[0]) + log.Debug().Interface("initDoc", string(initDoc)).Msg("generating init record") - return nil - }) - util.Fatal(err, "fix number transaction") - } -} + util.Fatal(err, "init record generation") -func guaranteeFloatsInFirstRecord(ctx context.Context, coll string, docs []json.RawMessage) { if !FirstRecord { return } cnt, err := client.GetDB().Count(ctx, coll, driver.Filter("{}")) - - var ep *driver.Error - if err != nil { + var ep *driver.Error if errors.As(err, &ep) && ep.Code == api.Code_NOT_FOUND { - log.Debug().Msg("collection doesn't exits, skip fixing numbers") + log.Debug().Msg("collection doesn't exits, skipping init record") return } util.Fatal(err, "get count") } - if cnt == 0 { - schema.DetectArrayOfObjects = true - schema.ReplaceNumber = true + if cnt != 0 { + log.Debug().Msg("collection is not empty, skipping init record") - fixNumbers(ctx, coll, docs) + FirstRecord = false - schema.DetectArrayOfObjects = false - schema.ReplaceNumber = false + return } + err = client.Transact(ctx, config.GetProjectName(), func(ctx context.Context, tx driver.Tx) error { + _, err = tx.Insert(ctx, coll, []driver.Document{initDoc}) + util.Fatal(err, "insert init record") + + _, err = tx.Delete(ctx, coll, driver.Filter("{}")) + util.Fatal(err, "delete init record") + + return nil + }) + util.Fatal(err, "init record transaction") + FirstRecord = false } func insertWithInference(ctx context.Context, coll string, docs []json.RawMessage) error { // FIXME: This is temporary fix, should moved to server ASAP - guaranteeFloatsInFirstRecord(ctx, coll, docs) + writeInitRecord(ctx, coll, docs) ptr := unsafe.Pointer(&docs) diff --git a/schema/inference.go b/schema/inference.go index 4803d17..ceac080 100644 --- a/schema/inference.go +++ b/schema/inference.go @@ -55,10 +55,7 @@ var ( ErrExpectedNumber = fmt.Errorf("expected json.Number") ErrUnsupportedType = fmt.Errorf("unsupported type") - HasArrayOfObjects bool - DummyRecord map[string]any - DetectArrayOfObjects bool - ReplaceNumber bool + HasArrayOfObjects bool ) func newInompatibleSchemaError(name, oldType, oldFormat, newType, newFormat string) error { @@ -257,11 +254,9 @@ func traverseArray(name string, existingField *schema.Field, newField *schema.Fi newField.Items.Format = nf if t == typeObject { - if DetectArrayOfObjects { - log.Debug().Msg("detected array of objects") + log.Debug().Msg("detected array of objects") - HasArrayOfObjects = true - } + HasArrayOfObjects = true values, _ := reflect.ValueOf(v).Index(i).Interface().(map[string]any) if err = traverseObject(name, newField.Items, newField.Items, values); err != nil { @@ -349,10 +344,6 @@ func traverseFields(sch map[string]*schema.Field, fields map[string]any, autoGen continue } - if sch != nil && sch[name] != nil && sch[name].Type == typeNumber && ReplaceNumber { - fields[name] = 1.1 - } - setAutoGenerate(autoGen, name, f) sch[name] = f @@ -391,10 +382,6 @@ func docToSchema(sch *schema.Schema, name string, data []byte, pk []string, auto sch.PrimaryKey = []string{"id"} } - if ReplaceNumber { - DummyRecord = m - } - return nil } @@ -410,3 +397,56 @@ func Infer(sch *schema.Schema, name string, docs []json.RawMessage, primaryKey [ return nil } + +func GenerateInitDoc(sch *schema.Schema, doc json.RawMessage) ([]byte, error) { + if sch.Fields == nil { + return nil, nil + } + + var initDoc map[string]interface{} + + dec := json.NewDecoder(bytes.NewBuffer(doc)) + dec.UseNumber() + + if err := dec.Decode(&initDoc); err != nil { + return nil, err + } + + for name := range sch.Fields { + if err := initDocTraverseFields(sch.Fields[name], initDoc, name); err != nil { + log.Debug().Err(err).Msg("init doc traverse fields") + return nil, err + } + } + + return json.Marshal(initDoc) +} + +func initDocTraverseFields(field *schema.Field, doc map[string]any, fieldName string) error { + switch field.Type { + case typeNumber: + doc[fieldName] = 0.0000001 + case typeObject: + vo := map[string]any{} + for name := range field.Fields { + if err := initDocTraverseFields(field.Fields[name], vo, name); err != nil { + return err + } + } + + doc[fieldName] = vo + case typeArray: + if field.Items.Type == typeObject { + vo := map[string]any{} + for name := range field.Items.Fields { + if err := initDocTraverseFields(field.Items.Fields[name], vo, name); err != nil { + return err + } + } + + doc[fieldName] = []map[string]any{vo} + } + } + + return nil +} From ae8d3e42140b5b4e1dadb5273a5df58b99b82117 Mon Sep 17 00:00:00 2001 From: Ovais Tariq Date: Sat, 13 May 2023 19:54:03 -0700 Subject: [PATCH 8/9] fix: Generate init record only for the first batch (#268) --- cmd/import.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/import.go b/cmd/import.go index a7d7862..f391442 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -80,11 +80,6 @@ func evolveSchema(ctx context.Context, db string, coll string, docs []json.RawMe } func writeInitRecord(ctx context.Context, coll string, docs []json.RawMessage) { - initDoc, err := schema.GenerateInitDoc(&sch, docs[0]) - log.Debug().Interface("initDoc", string(initDoc)).Msg("generating init record") - - util.Fatal(err, "init record generation") - if !FirstRecord { return } @@ -108,6 +103,11 @@ func writeInitRecord(ctx context.Context, coll string, docs []json.RawMessage) { return } + initDoc, err := schema.GenerateInitDoc(&sch, docs[0]) + log.Debug().Interface("initDoc", string(initDoc)).Msg("generating init record") + + util.Fatal(err, "init record generation") + err = client.Transact(ctx, config.GetProjectName(), func(ctx context.Context, tx driver.Tx) error { _, err = tx.Insert(ctx, coll, []driver.Document{initDoc}) util.Fatal(err, "insert init record") From 772f0eab0a3c7fdff6aa0cbd893c77ce3ea9260b Mon Sep 17 00:00:00 2001 From: Phil Leggetter Date: Mon, 15 May 2023 18:08:37 +0100 Subject: [PATCH 9/9] chore: remove shebang from bin/tigris placeholder (#271) --- pkg/npm/bin/tigris | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/npm/bin/tigris b/pkg/npm/bin/tigris index de49414..67dd248 100644 --- a/pkg/npm/bin/tigris +++ b/pkg/npm/bin/tigris @@ -1,4 +1 @@ -#!/ usr / bin / env node - -/* eslint-disable no-console */ -console.error('This is placeholder, which should be replaced by installation process'); +# This is placeholder, which should be replaced by installation process