Merge pull request #272 from tigrisdata/main

Beta release

cmd/import.go

@@ -17,6 +17,7 @@ package cmd
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"unsafe"
 
@@ -54,6 +55,10 @@ var (
 	ErrCollectionShouldExist = fmt.Errorf("collection should exist to import CSV with no field names")
 	ErrNoAppend              = fmt.Errorf(
 		"collection exists. use --append if you need to add documents to existing collection")
+
+	ErrNoRecordsExpected = fmt.Errorf("no records expected in the collection after fixing numbers")
+
+	FirstRecord = true
 )
 
 func evolveSchema(ctx context.Context, db string, coll string, docs []json.RawMessage) error {
@@ -74,7 +79,53 @@ func evolveSchema(ctx context.Context, db string, coll string, docs []json.RawMe
 	return util.Error(err, "create or update collection")
 }
 
+func writeInitRecord(ctx context.Context, coll string, docs []json.RawMessage) {
+	if !FirstRecord {
+		return
+	}
+
+	cnt, err := client.GetDB().Count(ctx, coll, driver.Filter("{}"))
+	if err != nil {
+		var ep *driver.Error
+		if errors.As(err, &ep) && ep.Code == api.Code_NOT_FOUND {
+			log.Debug().Msg("collection doesn't exits, skipping init record")
+			return
+		}
+
+		util.Fatal(err, "get count")
+	}
+
+	if cnt != 0 {
+		log.Debug().Msg("collection is not empty, skipping init record")
+
+		FirstRecord = false
+
+		return
+	}
+
+	initDoc, err := schema.GenerateInitDoc(&sch, docs[0])
+	log.Debug().Interface("initDoc", string(initDoc)).Msg("generating init record")
+
+	util.Fatal(err, "init record generation")
+
+	err = client.Transact(ctx, config.GetProjectName(), func(ctx context.Context, tx driver.Tx) error {
+		_, err = tx.Insert(ctx, coll, []driver.Document{initDoc})
+		util.Fatal(err, "insert init record")
+
+		_, err = tx.Delete(ctx, coll, driver.Filter("{}"))
+		util.Fatal(err, "delete init record")
+
+		return nil
+	})
+	util.Fatal(err, "init record transaction")
+
+	FirstRecord = false
+}
+
 func insertWithInference(ctx context.Context, coll string, docs []json.RawMessage) error {
+	// FIXME: This is temporary fix, should moved to server ASAP
+	writeInitRecord(ctx, coll, docs)
+
 	ptr := unsafe.Pointer(&docs)
 
 	_, err := client.GetDB().Insert(ctx, coll, *(*[]driver.Document)(ptr))

go.mod

@@ -19,9 +19,9 @@ require (
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/viper v1.15.0
 	github.com/stretchr/testify v1.8.2
-	github.com/tigrisdata/tigris-client-go v1.0.0-beta.37
+	github.com/tigrisdata/tigris-client-go v1.0.0
 	golang.org/x/net v0.10.0
-	golang.org/x/oauth2 v0.7.0
+	golang.org/x/oauth2 v0.8.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
@@ -36,7 +36,7 @@ require (
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/deepmap/oapi-codegen v1.12.4 // indirect
-	github.com/docker/distribution v2.8.1+incompatible // indirect
+	github.com/docker/distribution v2.8.2+incompatible // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/go-git/gcfg v1.5.0 // indirect
@@ -79,12 +79,12 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.4.2 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
-	golang.org/x/crypto v0.8.0 // indirect
+	golang.org/x/crypto v0.9.0 // indirect
 	golang.org/x/mod v0.10.0 // indirect
 	golang.org/x/sys v0.8.0 // indirect
 	golang.org/x/term v0.8.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
-	golang.org/x/tools v0.8.0 // indirect
+	golang.org/x/tools v0.9.1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
 	google.golang.org/grpc v1.55.0 // indirect

go.sum

@@ -92,8 +92,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/deepmap/oapi-codegen v1.12.4 h1:pPmn6qI9MuOtCz82WY2Xaw46EQjgvxednXXrP7g5Q2s=
 github.com/deepmap/oapi-codegen v1.12.4/go.mod h1:3lgHGMu6myQ2vqbbTXH2H1o4eXFTGnFiDaOaKKl5yas=
-github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68=
-github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
+github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/docker v23.0.6+incompatible h1:aBD4np894vatVX99UTx/GyOUOK4uEcROwA3+bQhEcoU=
 github.com/docker/docker v23.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
@@ -355,8 +355,8 @@ github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8=
 github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
-github.com/tigrisdata/tigris-client-go v1.0.0-beta.37 h1:YFwjiEiexMDHTudzdrfGk9I7TAIj/mamsp/XbfNFCrE=
-github.com/tigrisdata/tigris-client-go v1.0.0-beta.37/go.mod h1:2n6TQUdoTbzuTtakHT/ZNuK5X+I/i57BqqCcYAzG7y4=
+github.com/tigrisdata/tigris-client-go v1.0.0 h1:07Qw8Tm0qL15WiadP0hp4iBiRzfNSJ+GH4/ozO0nNs0=
+github.com/tigrisdata/tigris-client-go v1.0.0/go.mod h1:2n6TQUdoTbzuTtakHT/ZNuK5X+I/i57BqqCcYAzG7y4=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@@ -395,8 +395,8 @@ golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
 golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
-golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ=
-golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE=
+golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g=
+golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -490,8 +490,8 @@ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ
 golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
 golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk=
-golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g=
-golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4=
+golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8=
+golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -504,8 +504,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -649,8 +649,8 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y=
-golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
+golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
+golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

login/login.go

@@ -48,6 +48,12 @@ type instance struct {
 var (
 	ErrStateMismatched  = fmt.Errorf("state is not matched")
 	ErrInstanceNotFound = fmt.Errorf("instance not found")
+
+	defaultInstance = instance{
+		clientID: "GS8PrHA1aYblUR73yitqomc40ZYZ81jF",
+		authHost: "https://auth.tigrisdata.cloud/",
+		audience: "https://tigris-api-prod",
+	}
 )
 
 var (
@@ -295,7 +301,11 @@ func CmdLow(_ context.Context, host string) error {
 
 	inst, ok := instances[host]
 	if !ok {
-		return util.Error(fmt.Errorf("%w: %s", ErrInstanceNotFound, host), "Instance config not found")
+		if !strings.HasSuffix(host, ".tigrisdata.cloud") {
+			return util.Error(fmt.Errorf("%w: %s", ErrInstanceNotFound, host), "Instance config not found")
+		}
+
+		inst = defaultInstance
 	}
 
 	p, err := oidc.NewProvider(context.Background(), inst.authHost)

pkg/npm/bin/tigris

@@ -1,4 +1 @@
-#!/ usr / bin / env node
-
-/* eslint-disable no-console */
-console.error('This is placeholder, which should be replaced by installation process');
+# This is placeholder, which should be replaced by installation process

schema/inference.go

@@ -54,6 +54,8 @@ var (
 	ErrExpectedString     = fmt.Errorf("expected string type")
 	ErrExpectedNumber     = fmt.Errorf("expected json.Number")
 	ErrUnsupportedType    = fmt.Errorf("unsupported type")
+
+	HasArrayOfObjects bool
 )
 
 func newInompatibleSchemaError(name, oldType, oldFormat, newType, newFormat string) error {
@@ -71,15 +73,14 @@ func parseDateTime(s string) bool {
 	return false
 }
 
-func parseNumber(v any) (string, string, error) {
+func parseNumber(v any, existing *schema.Field) (string, string, error) {
 	n, ok := v.(json.Number)
 	if !ok {
 		return "", "", ErrExpectedNumber
 	}
 
-	if _, err := n.Int64(); err != nil || !DetectIntegers {
-		_, err = n.Float64()
-		if err != nil {
+	if _, err := n.Int64(); err != nil || (!DetectIntegers && (existing == nil || existing.Type != typeInteger)) {
+		if _, err = n.Float64(); err != nil {
 			return "", "", err
 		}
 
@@ -89,27 +90,31 @@ func parseNumber(v any) (string, string, error) {
 	return typeInteger, "", nil
 }
 
-func translateStringType(v interface{}) (string, string, error) {
+func needNarrowing(detect bool, existing *schema.Field, format string) bool {
+	return detect || existing != nil && existing.Format == format
+}
+
+func translateStringType(v any, existing *schema.Field) (string, string, error) {
 	t := reflect.TypeOf(v)
 
 	if t.PkgPath() == "encoding/json" && t.Name() == "Number" {
-		return parseNumber(v)
+		return parseNumber(v, existing)
 	}
 
 	s, ok := v.(string)
 	if !ok {
 		return "", "", ErrExpectedString
 	}
 
-	if parseDateTime(s) && DetectTimes {
+	if parseDateTime(s) && needNarrowing(DetectTimes, existing, formatDateTime) {
 		return typeString, formatDateTime, nil
 	}
 
-	if _, err := uuid.Parse(s); err == nil && DetectUUIDs {
+	if _, err := uuid.Parse(s); err == nil && needNarrowing(DetectUUIDs, existing, formatUUID) {
 		return typeString, formatUUID, nil
 	}
 
-	if len(s) != 0 && DetectByteArrays {
+	if len(s) != 0 && needNarrowing(DetectByteArrays, existing, formatByte) {
 		b := make([]byte, base64.StdEncoding.DecodedLen(len(s)))
 		if _, err := base64.StdEncoding.Decode(b, []byte(s)); err == nil {
 			return typeString, formatByte, nil
@@ -119,7 +124,7 @@ func translateStringType(v interface{}) (string, string, error) {
 	return typeString, "", nil
 }
 
-func translateType(v interface{}) (string, string, error) {
+func translateType(v any, existing *schema.Field) (string, string, error) {
 	t := reflect.TypeOf(v)
 
 	//nolint:golint,exhaustive
@@ -129,7 +134,7 @@ func translateType(v interface{}) (string, string, error) {
 	case reflect.Float64:
 		return typeNumber, "", nil
 	case reflect.String:
-		return translateStringType(v)
+		return translateStringType(v, existing)
 	case reflect.Slice, reflect.Array:
 		return typeArray, "", nil
 	case reflect.Map:
@@ -221,7 +226,7 @@ func traverseObject(name string, existingField *schema.Field, newField *schema.F
 
 func traverseArray(name string, existingField *schema.Field, newField *schema.Field, v any) error {
 	for i := 0; i < reflect.ValueOf(v).Len(); i++ {
-		t, format, err := translateType(reflect.ValueOf(v).Index(i).Interface())
+		t, format, err := translateType(reflect.ValueOf(v).Index(i).Interface(), existingField)
 		if err != nil {
 			return err
 		}
@@ -249,6 +254,10 @@ func traverseArray(name string, existingField *schema.Field, newField *schema.Fi
 		newField.Items.Format = nf
 
 		if t == typeObject {
+			log.Debug().Msg("detected array of objects")
+
+			HasArrayOfObjects = true
+
 			values, _ := reflect.ValueOf(v).Index(i).Interface().(map[string]any)
 			if err = traverseObject(name, newField.Items, newField.Items, values); err != nil {
 				return err
@@ -319,7 +328,7 @@ func traverseFields(sch map[string]*schema.Field, fields map[string]any, autoGen
 			continue
 		}
 
-		t, format, err := translateType(val)
+		t, format, err := translateType(val, sch[name])
 		if err != nil {
 			return err
 		}
@@ -388,3 +397,56 @@ func Infer(sch *schema.Schema, name string, docs []json.RawMessage, primaryKey [
 
 	return nil
 }
+
+func GenerateInitDoc(sch *schema.Schema, doc json.RawMessage) ([]byte, error) {
+	if sch.Fields == nil {
+		return nil, nil
+	}
+
+	var initDoc map[string]interface{}
+
+	dec := json.NewDecoder(bytes.NewBuffer(doc))
+	dec.UseNumber()
+
+	if err := dec.Decode(&initDoc); err != nil {
+		return nil, err
+	}
+
+	for name := range sch.Fields {
+		if err := initDocTraverseFields(sch.Fields[name], initDoc, name); err != nil {
+			log.Debug().Err(err).Msg("init doc traverse fields")
+			return nil, err
+		}
+	}
+
+	return json.Marshal(initDoc)
+}
+
+func initDocTraverseFields(field *schema.Field, doc map[string]any, fieldName string) error {
+	switch field.Type {
+	case typeNumber:
+		doc[fieldName] = 0.0000001
+	case typeObject:
+		vo := map[string]any{}
+		for name := range field.Fields {
+			if err := initDocTraverseFields(field.Fields[name], vo, name); err != nil {
+				return err
+			}
+		}
+
+		doc[fieldName] = vo
+	case typeArray:
+		if field.Items.Type == typeObject {
+			vo := map[string]any{}
+			for name := range field.Items.Fields {
+				if err := initDocTraverseFields(field.Items.Fields[name], vo, name); err != nil {
+					return err
+				}
+			}
+
+			doc[fieldName] = []map[string]any{vo}
+		}
+	}
+
+	return nil
+}

util/util.go

@@ -114,7 +114,7 @@ func Infof(format string, args ...interface{}) {
 }
 
 func Error(err error, msg string, args ...interface{}) error {
-	log.Err(err).CallerSkipFrame(3).Msgf(msg, args...)
+	log.Err(err).CallerSkipFrame(2).Msgf(msg, args...)
 
 	if err == nil {
 		return nil