diff --git a/cmd/import.go b/cmd/import.go index b8d2d5c..f391442 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -17,6 +17,7 @@ package cmd import ( "context" "encoding/json" + "errors" "fmt" "unsafe" @@ -54,6 +55,10 @@ var ( ErrCollectionShouldExist = fmt.Errorf("collection should exist to import CSV with no field names") ErrNoAppend = fmt.Errorf( "collection exists. use --append if you need to add documents to existing collection") + + ErrNoRecordsExpected = fmt.Errorf("no records expected in the collection after fixing numbers") + + FirstRecord = true ) func evolveSchema(ctx context.Context, db string, coll string, docs []json.RawMessage) error { @@ -74,7 +79,53 @@ func evolveSchema(ctx context.Context, db string, coll string, docs []json.RawMe return util.Error(err, "create or update collection") } +func writeInitRecord(ctx context.Context, coll string, docs []json.RawMessage) { + if !FirstRecord { + return + } + + cnt, err := client.GetDB().Count(ctx, coll, driver.Filter("{}")) + if err != nil { + var ep *driver.Error + if errors.As(err, &ep) && ep.Code == api.Code_NOT_FOUND { + log.Debug().Msg("collection doesn't exits, skipping init record") + return + } + + util.Fatal(err, "get count") + } + + if cnt != 0 { + log.Debug().Msg("collection is not empty, skipping init record") + + FirstRecord = false + + return + } + + initDoc, err := schema.GenerateInitDoc(&sch, docs[0]) + log.Debug().Interface("initDoc", string(initDoc)).Msg("generating init record") + + util.Fatal(err, "init record generation") + + err = client.Transact(ctx, config.GetProjectName(), func(ctx context.Context, tx driver.Tx) error { + _, err = tx.Insert(ctx, coll, []driver.Document{initDoc}) + util.Fatal(err, "insert init record") + + _, err = tx.Delete(ctx, coll, driver.Filter("{}")) + util.Fatal(err, "delete init record") + + return nil + }) + util.Fatal(err, "init record transaction") + + FirstRecord = false +} + func insertWithInference(ctx context.Context, coll string, docs []json.RawMessage) error { + // FIXME: This is temporary fix, should moved to server ASAP + writeInitRecord(ctx, coll, docs) + ptr := unsafe.Pointer(&docs) _, err := client.GetDB().Insert(ctx, coll, *(*[]driver.Document)(ptr)) diff --git a/go.mod b/go.mod index 2ec2453..2150e75 100644 --- a/go.mod +++ b/go.mod @@ -19,9 +19,9 @@ require ( github.com/spf13/cobra v1.7.0 github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.2 - github.com/tigrisdata/tigris-client-go v1.0.0-beta.37 + github.com/tigrisdata/tigris-client-go v1.0.0 golang.org/x/net v0.10.0 - golang.org/x/oauth2 v0.7.0 + golang.org/x/oauth2 v0.8.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -36,7 +36,7 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/deepmap/oapi-codegen v1.12.4 // indirect - github.com/docker/distribution v2.8.1+incompatible // indirect + github.com/docker/distribution v2.8.2+incompatible // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-git/gcfg v1.5.0 // indirect @@ -79,12 +79,12 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect - golang.org/x/crypto v0.8.0 // indirect + golang.org/x/crypto v0.9.0 // indirect golang.org/x/mod v0.10.0 // indirect golang.org/x/sys v0.8.0 // indirect golang.org/x/term v0.8.0 // indirect golang.org/x/text v0.9.0 // indirect - golang.org/x/tools v0.8.0 // indirect + golang.org/x/tools v0.9.1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/grpc v1.55.0 // indirect diff --git a/go.sum b/go.sum index 7e1551e..b99f7ad 100644 --- a/go.sum +++ b/go.sum @@ -92,8 +92,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deepmap/oapi-codegen v1.12.4 h1:pPmn6qI9MuOtCz82WY2Xaw46EQjgvxednXXrP7g5Q2s= github.com/deepmap/oapi-codegen v1.12.4/go.mod h1:3lgHGMu6myQ2vqbbTXH2H1o4eXFTGnFiDaOaKKl5yas= -github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= -github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v23.0.6+incompatible h1:aBD4np894vatVX99UTx/GyOUOK4uEcROwA3+bQhEcoU= github.com/docker/docker v23.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= @@ -355,8 +355,8 @@ github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= -github.com/tigrisdata/tigris-client-go v1.0.0-beta.37 h1:YFwjiEiexMDHTudzdrfGk9I7TAIj/mamsp/XbfNFCrE= -github.com/tigrisdata/tigris-client-go v1.0.0-beta.37/go.mod h1:2n6TQUdoTbzuTtakHT/ZNuK5X+I/i57BqqCcYAzG7y4= +github.com/tigrisdata/tigris-client-go v1.0.0 h1:07Qw8Tm0qL15WiadP0hp4iBiRzfNSJ+GH4/ozO0nNs0= +github.com/tigrisdata/tigris-client-go v1.0.0/go.mod h1:2n6TQUdoTbzuTtakHT/ZNuK5X+I/i57BqqCcYAzG7y4= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -395,8 +395,8 @@ golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= -golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -490,8 +490,8 @@ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -504,8 +504,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -649,8 +649,8 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= -golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/login/login.go b/login/login.go index f04d4b3..5220177 100644 --- a/login/login.go +++ b/login/login.go @@ -48,6 +48,12 @@ type instance struct { var ( ErrStateMismatched = fmt.Errorf("state is not matched") ErrInstanceNotFound = fmt.Errorf("instance not found") + + defaultInstance = instance{ + clientID: "GS8PrHA1aYblUR73yitqomc40ZYZ81jF", + authHost: "https://auth.tigrisdata.cloud/", + audience: "https://tigris-api-prod", + } ) var ( @@ -295,7 +301,11 @@ func CmdLow(_ context.Context, host string) error { inst, ok := instances[host] if !ok { - return util.Error(fmt.Errorf("%w: %s", ErrInstanceNotFound, host), "Instance config not found") + if !strings.HasSuffix(host, ".tigrisdata.cloud") { + return util.Error(fmt.Errorf("%w: %s", ErrInstanceNotFound, host), "Instance config not found") + } + + inst = defaultInstance } p, err := oidc.NewProvider(context.Background(), inst.authHost) diff --git a/pkg/npm/bin/tigris b/pkg/npm/bin/tigris index de49414..67dd248 100644 --- a/pkg/npm/bin/tigris +++ b/pkg/npm/bin/tigris @@ -1,4 +1 @@ -#!/ usr / bin / env node - -/* eslint-disable no-console */ -console.error('This is placeholder, which should be replaced by installation process'); +# This is placeholder, which should be replaced by installation process diff --git a/schema/inference.go b/schema/inference.go index d7f564e..ceac080 100644 --- a/schema/inference.go +++ b/schema/inference.go @@ -54,6 +54,8 @@ var ( ErrExpectedString = fmt.Errorf("expected string type") ErrExpectedNumber = fmt.Errorf("expected json.Number") ErrUnsupportedType = fmt.Errorf("unsupported type") + + HasArrayOfObjects bool ) func newInompatibleSchemaError(name, oldType, oldFormat, newType, newFormat string) error { @@ -71,15 +73,14 @@ func parseDateTime(s string) bool { return false } -func parseNumber(v any) (string, string, error) { +func parseNumber(v any, existing *schema.Field) (string, string, error) { n, ok := v.(json.Number) if !ok { return "", "", ErrExpectedNumber } - if _, err := n.Int64(); err != nil || !DetectIntegers { - _, err = n.Float64() - if err != nil { + if _, err := n.Int64(); err != nil || (!DetectIntegers && (existing == nil || existing.Type != typeInteger)) { + if _, err = n.Float64(); err != nil { return "", "", err } @@ -89,11 +90,15 @@ func parseNumber(v any) (string, string, error) { return typeInteger, "", nil } -func translateStringType(v interface{}) (string, string, error) { +func needNarrowing(detect bool, existing *schema.Field, format string) bool { + return detect || existing != nil && existing.Format == format +} + +func translateStringType(v any, existing *schema.Field) (string, string, error) { t := reflect.TypeOf(v) if t.PkgPath() == "encoding/json" && t.Name() == "Number" { - return parseNumber(v) + return parseNumber(v, existing) } s, ok := v.(string) @@ -101,15 +106,15 @@ func translateStringType(v interface{}) (string, string, error) { return "", "", ErrExpectedString } - if parseDateTime(s) && DetectTimes { + if parseDateTime(s) && needNarrowing(DetectTimes, existing, formatDateTime) { return typeString, formatDateTime, nil } - if _, err := uuid.Parse(s); err == nil && DetectUUIDs { + if _, err := uuid.Parse(s); err == nil && needNarrowing(DetectUUIDs, existing, formatUUID) { return typeString, formatUUID, nil } - if len(s) != 0 && DetectByteArrays { + if len(s) != 0 && needNarrowing(DetectByteArrays, existing, formatByte) { b := make([]byte, base64.StdEncoding.DecodedLen(len(s))) if _, err := base64.StdEncoding.Decode(b, []byte(s)); err == nil { return typeString, formatByte, nil @@ -119,7 +124,7 @@ func translateStringType(v interface{}) (string, string, error) { return typeString, "", nil } -func translateType(v interface{}) (string, string, error) { +func translateType(v any, existing *schema.Field) (string, string, error) { t := reflect.TypeOf(v) //nolint:golint,exhaustive @@ -129,7 +134,7 @@ func translateType(v interface{}) (string, string, error) { case reflect.Float64: return typeNumber, "", nil case reflect.String: - return translateStringType(v) + return translateStringType(v, existing) case reflect.Slice, reflect.Array: return typeArray, "", nil case reflect.Map: @@ -221,7 +226,7 @@ func traverseObject(name string, existingField *schema.Field, newField *schema.F func traverseArray(name string, existingField *schema.Field, newField *schema.Field, v any) error { for i := 0; i < reflect.ValueOf(v).Len(); i++ { - t, format, err := translateType(reflect.ValueOf(v).Index(i).Interface()) + t, format, err := translateType(reflect.ValueOf(v).Index(i).Interface(), existingField) if err != nil { return err } @@ -249,6 +254,10 @@ func traverseArray(name string, existingField *schema.Field, newField *schema.Fi newField.Items.Format = nf if t == typeObject { + log.Debug().Msg("detected array of objects") + + HasArrayOfObjects = true + values, _ := reflect.ValueOf(v).Index(i).Interface().(map[string]any) if err = traverseObject(name, newField.Items, newField.Items, values); err != nil { return err @@ -319,7 +328,7 @@ func traverseFields(sch map[string]*schema.Field, fields map[string]any, autoGen continue } - t, format, err := translateType(val) + t, format, err := translateType(val, sch[name]) if err != nil { return err } @@ -388,3 +397,56 @@ func Infer(sch *schema.Schema, name string, docs []json.RawMessage, primaryKey [ return nil } + +func GenerateInitDoc(sch *schema.Schema, doc json.RawMessage) ([]byte, error) { + if sch.Fields == nil { + return nil, nil + } + + var initDoc map[string]interface{} + + dec := json.NewDecoder(bytes.NewBuffer(doc)) + dec.UseNumber() + + if err := dec.Decode(&initDoc); err != nil { + return nil, err + } + + for name := range sch.Fields { + if err := initDocTraverseFields(sch.Fields[name], initDoc, name); err != nil { + log.Debug().Err(err).Msg("init doc traverse fields") + return nil, err + } + } + + return json.Marshal(initDoc) +} + +func initDocTraverseFields(field *schema.Field, doc map[string]any, fieldName string) error { + switch field.Type { + case typeNumber: + doc[fieldName] = 0.0000001 + case typeObject: + vo := map[string]any{} + for name := range field.Fields { + if err := initDocTraverseFields(field.Fields[name], vo, name); err != nil { + return err + } + } + + doc[fieldName] = vo + case typeArray: + if field.Items.Type == typeObject { + vo := map[string]any{} + for name := range field.Items.Fields { + if err := initDocTraverseFields(field.Items.Fields[name], vo, name); err != nil { + return err + } + } + + doc[fieldName] = []map[string]any{vo} + } + } + + return nil +} diff --git a/util/util.go b/util/util.go index db70583..e1cc9eb 100644 --- a/util/util.go +++ b/util/util.go @@ -114,7 +114,7 @@ func Infof(format string, args ...interface{}) { } func Error(err error, msg string, args ...interface{}) error { - log.Err(err).CallerSkipFrame(3).Msgf(msg, args...) + log.Err(err).CallerSkipFrame(2).Msgf(msg, args...) if err == nil { return nil