diff --git a/go.mod b/go.mod index 296ab6deb..134f79b0e 100644 --- a/go.mod +++ b/go.mod @@ -8,10 +8,11 @@ require ( github.com/blugelabs/bluge v0.1.9 github.com/blugelabs/bluge_segment_api v0.2.0 github.com/blugelabs/query_string v0.3.0 + github.com/bwmarrin/snowflake v0.3.0 github.com/gin-contrib/cors v1.3.1 github.com/gin-gonic/gin v1.7.4 github.com/go-ego/gse v0.70.0 - github.com/google/uuid v1.3.0 + github.com/goccy/go-json v0.9.6 github.com/joho/godotenv v1.4.0 github.com/minio/minio-go/v7 v7.0.21 github.com/robfig/cron/v3 v3.0.0 @@ -59,12 +60,15 @@ require ( github.com/go-playground/universal-translator v0.17.0 // indirect github.com/go-playground/validator/v10 v10.4.1 // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/golang/snappy v0.0.1 // indirect + github.com/golang/snappy v0.0.3 // indirect + github.com/google/go-cmp v0.5.7 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/klauspost/compress v1.13.5 // indirect github.com/klauspost/cpuid v1.3.1 // indirect + github.com/kr/pretty v0.3.0 // indirect github.com/leodido/go-urn v1.2.0 // indirect github.com/mattn/go-isatty v0.0.12 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect @@ -82,6 +86,7 @@ require ( github.com/segmentio/backo-go v1.0.0 // indirect github.com/sirupsen/logrus v1.8.1 // indirect github.com/smartystreets/assertions v1.2.0 // indirect + github.com/stretchr/testify v1.6.1 // indirect github.com/tklauser/go-sysconf v0.3.9 // indirect github.com/tklauser/numcpus v0.3.0 // indirect github.com/ugorji/go/codec v1.1.7 // indirect diff --git a/go.sum b/go.sum index 4d9a385f2..715cf52be 100644 --- a/go.sum +++ b/go.sum @@ -104,6 +104,8 @@ github.com/blugelabs/query_string v0.3.0 h1:/2XMd/3A0lIo33STXMUon4khLcTTybrAcyC0 github.com/blugelabs/query_string v0.3.0/go.mod h1:H0YFWhYAf8/xcv1zoswoUC8kM/fE9L/KEfsgySsnhfs= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0= +github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= github.com/caio/go-tdigest v3.1.0+incompatible h1:uoVMJ3Q5lXmVLCCqaMGHLBWnbGoN6Lpu7OAUPR60cds= github.com/caio/go-tdigest v3.1.0+incompatible/go.mod h1:sHQM/ubZStBUmF1WbB8FAm8q9GjDajLC5T7ydxE3JHI= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -122,6 +124,7 @@ github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8Nz github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -166,6 +169,8 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+ github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/goccy/go-json v0.9.6 h1:5/4CtRQdtsX0sal8fdVhTaiMN01Ri8BExZZ8iRmHQ6E= +github.com/goccy/go-json v0.9.6/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= @@ -197,8 +202,9 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -210,8 +216,9 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -264,11 +271,13 @@ github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leesper/go_rng v0.0.0-20190531154944-a612b043e353 h1:X/79QL0b4YJVO5+OsPH9rF2u428CIrGL/jLmPsoOQQ4= github.com/leesper/go_rng v0.0.0-20190531154944-a612b043e353/go.mod h1:N0SVk0uhy+E1PZ3C9ctsPRlvOPAFPkCNlcPBDkt0N3U= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= @@ -331,6 +340,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E= github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.3.0 h1:6NjYksEUlhurdVehpc7S7dk6DAmcKv8V9gG0FsVN2U4= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -363,8 +374,9 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tklauser/go-sysconf v0.3.9 h1:JeUVdAOWhhxVcU6Eqr/ATFHgXk/mmiItdKeJPev3vTo= github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= @@ -697,6 +709,8 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/auth/createuser.go b/pkg/auth/createuser.go index 90fc92f30..c700d3c41 100644 --- a/pkg/auth/createuser.go +++ b/pkg/auth/createuser.go @@ -4,11 +4,11 @@ import ( "time" "github.com/blugelabs/bluge" - "github.com/google/uuid" "github.com/rs/zerolog/log" "golang.org/x/crypto/argon2" "github.com/zinclabs/zinc/pkg/core" + "github.com/zinclabs/zinc/pkg/ider" ) func CreateUser(userID, name, plaintextPassword, role string) (*ZincUser, error) { @@ -52,7 +52,7 @@ func CreateUser(userID, name, plaintextPassword, role string) (*ZincUser, error) usersIndexWriter := core.ZINC_SYSTEM_INDEX_LIST["_users"].Writer err = usersIndexWriter.Update(bdoc.ID(), bdoc) if err != nil { - log.Printf("error updating document: %v", err) + log.Printf("error updating document: %s", err.Error()) return nil, err } @@ -85,7 +85,7 @@ func GeneratePassword(password, salt string) string { } func GenerateSalt() string { - return uuid.New().String() + return ider.Generate() } type ZincUser struct { diff --git a/pkg/auth/deleteuser.go b/pkg/auth/deleteuser.go index e6272e266..9a4b81eca 100644 --- a/pkg/auth/deleteuser.go +++ b/pkg/auth/deleteuser.go @@ -13,7 +13,7 @@ func DeleteUser(userID string) bool { usersIndexWriter := core.ZINC_SYSTEM_INDEX_LIST["_users"].Writer err := usersIndexWriter.Delete(bdoc.ID()) if err != nil { - log.Printf("error deleting user: %v", err) + log.Printf("error deleting user: %s", err.Error()) return false } diff --git a/pkg/auth/getuser.go b/pkg/auth/getuser.go index 7c919320c..a221ecd65 100644 --- a/pkg/auth/getuser.go +++ b/pkg/auth/getuser.go @@ -18,7 +18,7 @@ func GetUser(userID string) (ZincUser, bool, error) { reader, _ := core.ZINC_SYSTEM_INDEX_LIST["_users"].Writer.Reader() dmi, err := reader.Search(context.Background(), searchRequest) if err != nil { - log.Printf("auth.GetUser: error executing search: %v", err) + log.Printf("auth.GetUser: error executing search: %s", err.Error()) return user, userExists, err } @@ -47,7 +47,7 @@ func GetUser(userID string) (ZincUser, bool, error) { return true }) if err != nil { - log.Printf("auth.GetUser: error accessing stored fields: %v", err) + log.Printf("auth.GetUser: error accessing stored fields: %s", err.Error()) return user, userExists, err } else { return user, userExists, nil diff --git a/pkg/auth/getusers.go b/pkg/auth/getusers.go index 55bd121fa..4c40b405c 100644 --- a/pkg/auth/getusers.go +++ b/pkg/auth/getusers.go @@ -19,7 +19,7 @@ func GetAllUsersWorker() (*v1.SearchResponse, error) { reader, _ := usersIndex.Writer.Reader() dmi, err := reader.Search(context.Background(), searchRequest) if err != nil { - log.Printf("error executing search: %v", err) + log.Printf("error executing search: %s", err.Error()) } var Hits []v1.Hit @@ -48,7 +48,7 @@ func GetAllUsersWorker() (*v1.SearchResponse, error) { return true }) if err != nil { - log.Printf("error accessing stored fields: %v", err) + log.Printf("error accessing stored fields: %s", err.Error()) } hit := v1.Hit{ diff --git a/pkg/core/index.go b/pkg/core/index.go index e5fa7a548..c40ec5f18 100644 --- a/pkg/core/index.go +++ b/pkg/core/index.go @@ -2,7 +2,6 @@ package core import ( "context" - "encoding/json" "fmt" "math" "path/filepath" @@ -12,6 +11,7 @@ import ( "github.com/blugelabs/bluge" "github.com/blugelabs/bluge/analysis" + "github.com/goccy/go-json" "github.com/rs/zerolog/log" meta "github.com/zinclabs/zinc/pkg/meta/v2" @@ -351,3 +351,7 @@ func (index *Index) GainDocsCount(n int64) { atomic.AddInt64(&index.DocsCount, n) index.ReLoadStorageSize() } + +func (index *Index) Close() error { + return index.Writer.Close() +} diff --git a/pkg/core/loadindexes.go b/pkg/core/loadindexes.go index b0d42e0b2..c5837ac49 100644 --- a/pkg/core/loadindexes.go +++ b/pkg/core/loadindexes.go @@ -2,7 +2,6 @@ package core import ( "context" - "encoding/json" "fmt" "os" @@ -10,6 +9,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/blugelabs/bluge" "github.com/blugelabs/bluge/analysis" + "github.com/goccy/go-json" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/rs/zerolog/log" @@ -47,7 +47,7 @@ func LoadZincIndexesFromMeta() (map[string]*Index, error) { dmi, err := reader.Search(context.Background(), searchRequest) if err != nil { - return nil, fmt.Errorf("core.LoadZincIndexesFromMeta: error executing search: %v", err) + return nil, fmt.Errorf("core.LoadZincIndexesFromMeta: error executing search: %s", err.Error()) } indexList := make(map[string]*Index) @@ -73,14 +73,14 @@ func LoadZincIndexesFromMeta() (map[string]*Index, error) { log.Info().Msgf("Loading user index... [%s:%s]", index.Name, index.StorageType) if err != nil { - log.Printf("core.LoadZincIndexesFromMeta: error accessing stored fields: %v", err) + log.Printf("core.LoadZincIndexesFromMeta: error accessing stored fields: %s", err.Error()) } // load index analysis if index.Settings != nil && index.Settings.Analysis != nil { index.CachedAnalyzers, err = zincanalysis.RequestAnalyzer(index.Settings.Analysis) if err != nil { - log.Printf("core.LoadZincIndexesFromMeta: error parse stored analysis: %v", err) + log.Printf("core.LoadZincIndexesFromMeta: error parse stored analysis: %s", err.Error()) } } @@ -91,7 +91,7 @@ func LoadZincIndexesFromMeta() (map[string]*Index, error) { } index.Writer, err = LoadIndexWriter(index.Name, index.StorageType, defaultSearchAnalyzer) if err != nil { - log.Error().Msgf("Loading user index... [%s:%s] error: %v", index.Name, index.StorageType, err) + log.Error().Msgf("Loading user index... [%s:%s] index writer error: %s", index.Name, index.StorageType, err.Error()) } // load index docs count @@ -235,3 +235,12 @@ func LoadZincIndexesFromMinIO() (map[string]*Index, error) { return IndexList, nil } + +func CloseIndexes() { + for _, index := range ZINC_INDEX_LIST { + _ = index.Close() + } + for _, index := range ZINC_SYSTEM_INDEX_LIST { + _ = index.Close() + } +} diff --git a/pkg/core/multi_search_v2.go b/pkg/core/multi_search_v2.go index 1ae6aa9e0..4c149283a 100644 --- a/pkg/core/multi_search_v2.go +++ b/pkg/core/multi_search_v2.go @@ -47,7 +47,7 @@ func MultiSearchV2(indexName string, query *meta.ZincQuery) (*meta.SearchRespons dmi, err := bluge.MultiSearch(ctx, searchRequest, readers...) if err != nil { - log.Printf("core.MultiSearchV2: error executing search: %v", err) + log.Printf("core.MultiSearchV2: error executing search: %s", err.Error()) if err == context.DeadlineExceeded { return &meta.SearchResponse{ TimedOut: true, diff --git a/pkg/core/newindex.go b/pkg/core/newindex.go index b50b2182c..b5cecc60b 100644 --- a/pkg/core/newindex.go +++ b/pkg/core/newindex.go @@ -1,13 +1,13 @@ package core import ( - "encoding/json" "fmt" "strings" "time" "github.com/blugelabs/bluge" "github.com/blugelabs/bluge/analysis" + "github.com/goccy/go-json" "github.com/zinclabs/zinc/pkg/directory" meta "github.com/zinclabs/zinc/pkg/meta/v2" @@ -15,9 +15,7 @@ import ( ) // NewIndex creates an instance of a physical zinc index that can be used to store and retrieve data. -func NewIndex( - name string, storageType string, useNewIndexMeta int, - defaultSearchAnalyzer *analysis.Analyzer) (*Index, error) { +func NewIndex(name, storageType string, useNewIndexMeta int, defaultSearchAnalyzer *analysis.Analyzer) (*Index, error) { if name == "" { return nil, fmt.Errorf("core.NewIndex: index name cannot be empty") } @@ -125,7 +123,7 @@ func StoreIndex(index *Index) error { err := ZINC_SYSTEM_INDEX_LIST["_index"].Writer.Update(bdoc.ID(), bdoc) if err != nil { - return fmt.Errorf("core.StoreIndex: index: %s, error: %v", index.Name, err) + return fmt.Errorf("core.StoreIndex: index: %s, error: %s", index.Name, err.Error()) } // cache index @@ -139,7 +137,7 @@ func DeleteIndex(name string) error { bdoc.AddField(bluge.NewCompositeFieldExcluding("_all", nil)) err := ZINC_SYSTEM_INDEX_LIST["_index"].Writer.Delete(bdoc.ID()) if err != nil { - return fmt.Errorf("core.DeleteIndex: error deleting template: %v", err) + return fmt.Errorf("core.DeleteIndex: error deleting template: %s", err.Error()) } return nil diff --git a/pkg/core/search.go b/pkg/core/search.go index 4dd027eac..e38184e0e 100644 --- a/pkg/core/search.go +++ b/pkg/core/search.go @@ -77,13 +77,13 @@ func (index *Index) Search(iQuery *v1.ZincQuery) (*v1.SearchResponse, error) { reader, err := index.Writer.Reader() if err != nil { - log.Printf("error accessing reader: %v", err) + log.Printf("error accessing reader: %s", err.Error()) } defer reader.Close() dmi, err := reader.Search(context.Background(), searchRequest) if err != nil { - log.Printf("error executing search: %v", err) + log.Printf("error executing search: %s", err.Error()) } var Hits []v1.Hit @@ -103,11 +103,10 @@ func (index *Index) Search(iQuery *v1.ZincQuery) (*v1.SearchResponse, error) { result = uquery.HandleSource(sourceCtl, value) default: } - return true }) if err != nil { - log.Printf("error accessing stored fields: %v", err) + log.Printf("error accessing stored fields: %s", err.Error()) } hit := v1.Hit{ @@ -123,7 +122,7 @@ func (index *Index) Search(iQuery *v1.ZincQuery) (*v1.SearchResponse, error) { next, err = dmi.Next() } if err != nil { - log.Printf("error iterating results: %v", err) + log.Printf("error iterating results: %s", err.Error()) } resp := &v1.SearchResponse{ @@ -140,7 +139,7 @@ func (index *Index) Search(iQuery *v1.ZincQuery) (*v1.SearchResponse, error) { if len(iQuery.Aggregations) > 0 { resp.Aggregations, err = uquery.ParseAggregations(dmi.Aggregations()) if err != nil { - log.Printf("error parse aggregation results: %v", err) + log.Printf("error parse aggregation results: %s", err.Error()) } if len(resp.Aggregations) > 0 { delete(resp.Aggregations, "count") diff --git a/pkg/core/search_v2.go b/pkg/core/search_v2.go index ef91493df..e05452c03 100644 --- a/pkg/core/search_v2.go +++ b/pkg/core/search_v2.go @@ -23,7 +23,7 @@ func (index *Index) SearchV2(query *meta.ZincQuery) (*meta.SearchResponse, error reader, err := index.Writer.Reader() if err != nil { - log.Printf("index.SearchV2: error accessing reader: %v", err) + log.Printf("index.SearchV2: error accessing reader: %s", err.Error()) return nil, err } defer reader.Close() @@ -37,7 +37,7 @@ func (index *Index) SearchV2(query *meta.ZincQuery) (*meta.SearchResponse, error dmi, err := reader.Search(ctx, searchRequest) if err != nil { - log.Printf("index.SearchV2: error executing search: %v", err) + log.Printf("index.SearchV2: error executing search: %s", err.Error()) if err == context.DeadlineExceeded { return &meta.SearchResponse{ TimedOut: true, @@ -110,7 +110,7 @@ func searchV2(dmi search.DocumentMatchIterator, query *meta.ZincQuery, mappings return true }) if err != nil { - log.Printf("core.SearchV2: error accessing stored fields: %v", err) + log.Printf("core.SearchV2: error accessing stored fields: %s", err.Error()) continue } @@ -129,7 +129,7 @@ func searchV2(dmi search.DocumentMatchIterator, query *meta.ZincQuery, mappings next, err = dmi.Next() } if err != nil { - log.Printf("core.SearchV2: error iterating results: %v", err) + log.Printf("core.SearchV2: error iterating results: %s", err.Error()) } resp.Took = int(dmi.Aggregations().Duration().Milliseconds()) @@ -143,7 +143,7 @@ func searchV2(dmi search.DocumentMatchIterator, query *meta.ZincQuery, mappings } if err := parser.FormatResponse(resp, query, dmi.Aggregations()); err != nil { - log.Printf("core.SearchV2: error format response: %v", err) + log.Printf("core.SearchV2: error format response: %s", err.Error()) } return resp, nil diff --git a/pkg/core/telemetry.go b/pkg/core/telemetry.go index 4715bedcf..1a71ec5c8 100644 --- a/pkg/core/telemetry.go +++ b/pkg/core/telemetry.go @@ -8,13 +8,13 @@ import ( "time" "github.com/blugelabs/bluge" - "github.com/google/uuid" "github.com/robfig/cron/v3" "github.com/rs/zerolog/log" "github.com/shirou/gopsutil/cpu" "github.com/shirou/gopsutil/mem" "gopkg.in/segmentio/analytics-go.v3" + "github.com/zinclabs/zinc/pkg/ider" v1 "github.com/zinclabs/zinc/pkg/meta/v1" "github.com/zinclabs/zinc/pkg/zutils" ) @@ -40,7 +40,7 @@ func newTelemetry() *telemetry { } func (t *telemetry) createInstanceID() string { - instanceID := uuid.New().String() + instanceID := ider.Generate() doc := bluge.NewDocument("instance_id") doc.AddField(bluge.NewKeywordField("value", instanceID).StoreValue()) ZINC_SYSTEM_INDEX_LIST["_metadata"].Writer.Update(doc.ID(), doc) @@ -58,7 +58,7 @@ func (t *telemetry) getInstanceID() string { reader, _ := ZINC_SYSTEM_INDEX_LIST["_metadata"].Writer.Reader() dmi, err := reader.Search(context.Background(), searchRequest) if err != nil { - log.Printf("core.Telemetry.GetInstanceID: error executing search: %v", err) + log.Printf("core.Telemetry.GetInstanceID: error executing search: %s", err.Error()) } next, err := dmi.Next() @@ -70,7 +70,7 @@ func (t *telemetry) getInstanceID() string { return true }) if err != nil { - log.Printf("core.Telemetry.GetInstanceID: error accessing stored fields: %v", err) + log.Printf("core.Telemetry.GetInstanceID: error accessing stored fields: %s", err.Error()) } } diff --git a/pkg/core/template.go b/pkg/core/template.go index ebcd96926..196db5238 100644 --- a/pkg/core/template.go +++ b/pkg/core/template.go @@ -2,13 +2,13 @@ package core import ( "context" - "encoding/json" "fmt" "regexp" "strings" "time" "github.com/blugelabs/bluge" + "github.com/goccy/go-json" "github.com/rs/zerolog/log" meta "github.com/zinclabs/zinc/pkg/meta/v2" @@ -28,7 +28,7 @@ func ListTemplates(pattern string) ([]IndexTemplate, error) { dmi, err := reader.Search(context.Background(), searchRequest) if err != nil { - return nil, fmt.Errorf("core.ListTemplates: error executing search: %v", err) + return nil, fmt.Errorf("core.ListTemplates: error executing search: %s", err.Error()) } templates := make([]IndexTemplate, 0) @@ -50,7 +50,7 @@ func ListTemplates(pattern string) ([]IndexTemplate, error) { return true }) if err != nil { - log.Printf("core.ListTemplates: error accessing stored fields: %v", err) + log.Printf("core.ListTemplates: error accessing stored fields: %s", err.Error()) } templates = append(templates, IndexTemplate{ @@ -108,7 +108,7 @@ func NewTemplate(name string, template *meta.Template) error { index := ZINC_SYSTEM_INDEX_LIST["_index_template"].Writer err := index.Update(bdoc.ID(), bdoc) if err != nil { - return fmt.Errorf("template: error updating document: %v", err) + return fmt.Errorf("template: error updating document: %s", err.Error()) } return nil @@ -127,13 +127,13 @@ func LoadTemplate(name string) (*meta.Template, bool, error) { dmi, err := reader.Search(context.Background(), searchRequest) if err != nil { - return nil, false, fmt.Errorf("template: error executing search: %v", err) + return nil, false, fmt.Errorf("template: error executing search: %s", err.Error()) } tpl := new(meta.Template) next, err := dmi.Next() if err != nil { - return nil, false, fmt.Errorf("template: error accessing stored fields: %v", err) + return nil, false, fmt.Errorf("template: error accessing stored fields: %s", err.Error()) } if next == nil { return nil, false, nil @@ -148,7 +148,7 @@ func LoadTemplate(name string) (*meta.Template, bool, error) { return true }) if err != nil { - return nil, false, fmt.Errorf("template: error accessing stored fields: %v", err) + return nil, false, fmt.Errorf("template: error accessing stored fields: %s", err.Error()) } return tpl, true, nil @@ -160,7 +160,7 @@ func DeleteTemplate(name string) error { bdoc.AddField(bluge.NewCompositeFieldExcluding("_all", nil)) err := ZINC_SYSTEM_INDEX_LIST["_index_template"].Writer.Delete(bdoc.ID()) if err != nil { - return fmt.Errorf("template: error deleting template: %v", err) + return fmt.Errorf("template: error deleting template: %s", err.Error()) } return nil @@ -175,7 +175,7 @@ func UseTemplate(indexName string) (*meta.Template, error) { dmi, err := reader.Search(context.Background(), searchRequest) if err != nil { - return nil, fmt.Errorf("core.UseTemplate: error executing search: %v", err) + return nil, fmt.Errorf("core.UseTemplate: error executing search: %s", err.Error()) } templates := make([]*meta.Template, 0) @@ -189,7 +189,7 @@ func UseTemplate(indexName string) (*meta.Template, error) { return true }) if err != nil { - log.Printf("core.UseTemplate: error accessing stored fields: %v", err) + log.Printf("core.UseTemplate: error accessing stored fields: %s", err.Error()) } templates = append(templates, tpl) diff --git a/pkg/core/updatedocument.go b/pkg/core/updatedocument.go index f59e3fcfa..abac56c5e 100644 --- a/pkg/core/updatedocument.go +++ b/pkg/core/updatedocument.go @@ -2,9 +2,7 @@ package core // UpdateDocument inserts or updates a document in the zinc index func (index *Index) UpdateDocument(docID string, doc map[string]interface{}, mintedID bool) error { - bdoc, err := index.BuildBlugeDocumentFromJSON(docID, doc) - if err != nil { return err } @@ -17,11 +15,5 @@ func (index *Index) UpdateDocument(docID string, doc map[string]interface{}, min err = writer.Insert(bdoc) index.GainDocsCount(1) } - // err = writer.Update(bdoc.ID(), bdoc) - if err != nil { - return err - } - - return nil - + return err } diff --git a/pkg/handlers/bulk.go b/pkg/handlers/bulk.go index 76118a08e..90f00acae 100644 --- a/pkg/handlers/bulk.go +++ b/pkg/handlers/bulk.go @@ -2,7 +2,6 @@ package handlers import ( "bufio" - "encoding/json" "errors" "io" "net/http" @@ -11,10 +10,11 @@ import ( "github.com/blugelabs/bluge" "github.com/blugelabs/bluge/index" "github.com/gin-gonic/gin" - "github.com/google/uuid" + "github.com/goccy/go-json" "github.com/rs/zerolog/log" "github.com/zinclabs/zinc/pkg/core" + "github.com/zinclabs/zinc/pkg/ider" "github.com/zinclabs/zinc/pkg/startup" ) @@ -64,9 +64,13 @@ func BulkHandlerWorker(target string, body io.ReadCloser) (*BulkResponse, error) var indexesInThisBatch []string var documentsInBatch int var doc map[string]interface{} + var err error for scanner.Scan() { // Read each line - if err := json.Unmarshal(scanner.Bytes(), &doc); err != nil { - log.Error().Msgf("bulk.json.Unmarshal: err %v", err) + for k := range doc { + delete(doc, k) + } + if err = json.Unmarshal(scanner.Bytes(), &doc); err != nil { + log.Error().Msgf("bulk.json.Unmarshal: err %s", err.Error()) continue } @@ -82,7 +86,7 @@ func BulkHandlerWorker(target string, body io.ReadCloser) (*BulkResponse, error) docID = val.(string) } if docID == "" { - docID = uuid.New().String() + docID = ider.Generate() mintedID = true } @@ -104,12 +108,6 @@ func BulkHandlerWorker(target string, body io.ReadCloser) (*BulkResponse, error) default: } - // Since this is a bulk request, we need to check if we already created a new batch for this index. We need to create 1 batch per index. - if DoesExistInThisRequest(indexesInThisBatch, indexName) == -1 { // Add the list of indexes to the batch if it's not already there - indexesInThisBatch = append(indexesInThisBatch, indexName) - batch[indexName] = index.NewBatch() - } - _, exists := core.GetIndex(indexName) if !exists { // If the requested indexName does not exist then create it newIndex, err := core.NewIndex(indexName, "disk", core.UseNewIndexMeta, nil) @@ -122,13 +120,17 @@ func BulkHandlerWorker(target string, body io.ReadCloser) (*BulkResponse, error) } } + // Since this is a bulk request, we need to check if we already created a new batch for this index. We need to create 1 batch per index. + if DoesExistInThisRequest(indexesInThisBatch, indexName) == -1 { // Add the list of indexes to the batch if it's not already there + indexesInThisBatch = append(indexesInThisBatch, indexName) + batch[indexName] = index.NewBatch() + } + bdoc, err := core.ZINC_INDEX_LIST[indexName].BuildBlugeDocumentFromJSON(docID, doc) if err != nil { return bulkRes, err } - documentsInBatch++ - // Add the documen to the batch. We will persist the batch to the index // when we have processed all documents in the request if !mintedID { @@ -137,18 +139,19 @@ func BulkHandlerWorker(target string, body io.ReadCloser) (*BulkResponse, error) batch[indexName].Insert(bdoc) } + documentsInBatch++ + // refresh index stats core.ZINC_INDEX_LIST[indexName].GainDocsCount(1) if documentsInBatch >= batchSize { - for _, indexN := range indexesInThisBatch { + for _, indexName := range indexesInThisBatch { // Persist the batch to the index - err := core.ZINC_INDEX_LIST[indexN].Writer.Batch(batch[indexN]) - if err != nil { - log.Error().Msgf("Error updating batch: %v", err) + if err := core.ZINC_INDEX_LIST[indexName].Writer.Batch(batch[indexName]); err != nil { + log.Error().Msgf("bulk: index updating batch err %s", err.Error()) return bulkRes, err } - batch[indexN].Reset() + batch[indexName].Reset() } documentsInBatch = 0 } @@ -202,18 +205,15 @@ func BulkHandlerWorker(target string, body io.ReadCloser) (*BulkResponse, error) return bulkRes, err } - for _, indexN := range indexesInThisBatch { - writer := core.ZINC_INDEX_LIST[indexN].Writer + for _, indexName := range indexesInThisBatch { // Persist the batch to the index - err := writer.Batch(batch[indexN]) - if err != nil { - log.Printf("Error updating batch: %v", err) + if err := core.ZINC_INDEX_LIST[indexName].Writer.Batch(batch[indexName]); err != nil { + log.Printf("bulk: index updating batch err %s", err.Error()) return bulkRes, err } } return bulkRes, nil - } // DoesExistInThisRequest takes a slice and looks for an element in it. If found it will diff --git a/pkg/handlers/updatedocument.go b/pkg/handlers/updatedocument.go index 61219ec87..cecb3cd19 100644 --- a/pkg/handlers/updatedocument.go +++ b/pkg/handlers/updatedocument.go @@ -4,17 +4,18 @@ import ( "net/http" "github.com/gin-gonic/gin" - "github.com/google/uuid" "github.com/zinclabs/zinc/pkg/core" + "github.com/zinclabs/zinc/pkg/ider" ) func UpdateDocument(c *gin.Context) { indexName := c.Param("target") queryID := c.Param("id") // ID for the document to be updated provided in URL path + var err error var doc map[string]interface{} - if err := c.BindJSON(&doc); err != nil { + if err = c.BindJSON(&doc); err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } @@ -29,11 +30,10 @@ func UpdateDocument(c *gin.Context) { docID = queryID } if docID == "" { - docID = uuid.New().String() // Generate a new ID if ID was not provided + docID = ider.Generate() mintedID = true } - var err error // If the index does not exist, then create it index, exists := core.GetIndex(indexName) if !exists { diff --git a/pkg/handlers/v2/search.go b/pkg/handlers/v2/search.go index f29c1f8ae..f7b62e3da 100644 --- a/pkg/handlers/v2/search.go +++ b/pkg/handlers/v2/search.go @@ -18,7 +18,7 @@ func SearchIndex(c *gin.Context) { query := new(meta.ZincQuery) if err := c.BindJSON(query); err != nil { - log.Printf("handlers.v2.SearchIndex: %v", err) + log.Printf("handlers.v2.SearchIndex: %s", err.Error()) c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) return } diff --git a/pkg/ider/id.go b/pkg/ider/id.go new file mode 100644 index 000000000..630b3f71e --- /dev/null +++ b/pkg/ider/id.go @@ -0,0 +1,28 @@ +package ider + +import ( + "os" + "strconv" + + "github.com/bwmarrin/snowflake" + "github.com/rs/zerolog/log" +) + +var node *snowflake.Node + +func init() { + var err error + nodeID := os.Getenv("ZINC_NODE_ID") + if nodeID == "" { + nodeID = "1" + } + id, _ := strconv.ParseInt(nodeID, 10, 64) + node, err = snowflake.NewNode(id) + if err != nil { + log.Fatal().Msgf("id generater init err %s", err.Error()) + } +} + +func Generate() string { + return node.Generate().String() +} diff --git a/pkg/uquery/aggregation.go b/pkg/uquery/aggregation.go index 5997fc8a4..b6e9f512f 100644 --- a/pkg/uquery/aggregation.go +++ b/pkg/uquery/aggregation.go @@ -33,7 +33,7 @@ func AddAggregations(req zincaggregation.SearchAggregation, aggs map[string]v1.A case "numeric": subreq = zincaggregation.NewTermsAggregation(search.Field(agg.Field), zincaggregation.NumericValueSource, agg.Size) default: - return fmt.Errorf("terms aggregation not supported type [%s:[%v]]", agg.Field, mapping.Properties[agg.Field].Type) + return fmt.Errorf("terms aggregation not supported type [%s:[%s]]", agg.Field, mapping.Properties[agg.Field].Type) } if len(agg.Aggregations) > 0 { if err := AddAggregations(subreq, agg.Aggregations, mapping); err != nil { @@ -85,7 +85,7 @@ func AddAggregations(req zincaggregation.SearchAggregation, aggs map[string]v1.A case "count": req.AddAggregation(name, aggregations.CountMatches()) default: - return fmt.Errorf("aggregation not supported %v", agg.AggType) + return fmt.Errorf("aggregation not supported %s", agg.AggType) } } diff --git a/pkg/uquery/querystring.go b/pkg/uquery/querystring.go index c96585dfb..2933793c4 100644 --- a/pkg/uquery/querystring.go +++ b/pkg/uquery/querystring.go @@ -14,7 +14,7 @@ func QueryStringQuery(iQuery *v1.ZincQuery) (bluge.SearchRequest, error) { options.WithDefaultAnalyzer(analyzer.NewStandardAnalyzer()) userQuery, err := querystr.ParseQueryString(iQuery.Query.Term, options) if err != nil { - return nil, fmt.Errorf("error parsing query string '%s': %v", iQuery.Query.Term, err) + return nil, fmt.Errorf("error parsing query string '%s': %s", iQuery.Query.Term, err.Error()) } dateQuery := bluge.NewDateRangeQuery(iQuery.Query.StartTime, iQuery.Query.EndTime).SetField("@timestamp") diff --git a/pkg/uquery/source.go b/pkg/uquery/source.go index 8a30c2924..3c2c685f0 100644 --- a/pkg/uquery/source.go +++ b/pkg/uquery/source.go @@ -1,7 +1,7 @@ package uquery import ( - "encoding/json" + "github.com/goccy/go-json" v1 "github.com/zinclabs/zinc/pkg/meta/v1" ) diff --git a/pkg/uquery/v2/aggregation/aggregation.go b/pkg/uquery/v2/aggregation/aggregation.go index d83bba76d..cf36401e7 100644 --- a/pkg/uquery/v2/aggregation/aggregation.go +++ b/pkg/uquery/v2/aggregation/aggregation.go @@ -51,7 +51,7 @@ func Request(req zincaggregation.SearchAggregation, aggs map[string]meta.Aggrega default: return errors.New( errors.ErrorTypeParsingException, - fmt.Sprintf("[terms] aggregation doesn't support values of type: [%s:[%v]]", agg.Terms.Field, mappings.Properties[agg.Terms.Field].Type), + fmt.Sprintf("[terms] aggregation doesn't support values of type: [%s:[%s]]", agg.Terms.Field, mappings.Properties[agg.Terms.Field].Type), ) } if len(agg.Aggregations) > 0 { @@ -93,7 +93,7 @@ func Request(req zincaggregation.SearchAggregation, aggs map[string]meta.Aggrega if agg.DateRange.TimeZone != "" { timeZone, err = zutils.ParseTimeZone(agg.DateRange.TimeZone) if err != nil { - return errors.New(errors.ErrorTypeXContentParseException, fmt.Sprintf("[date_range] time_zone parse err %v", err)) + return errors.New(errors.ErrorTypeXContentParseException, fmt.Sprintf("[date_range] time_zone parse err %s", err.Error())) } } switch mappings.Properties[agg.DateRange.Field].Type { @@ -105,13 +105,13 @@ func Request(req zincaggregation.SearchAggregation, aggs map[string]meta.Aggrega if v.From != "" { from, err = time.ParseInLocation(format, v.From, timeZone) if err != nil { - return errors.New(errors.ErrorTypeIllegalArgumentException, "[date_range] range value from parse error "+err.Error()) + return errors.New(errors.ErrorTypeIllegalArgumentException, fmt.Sprintf("[date_range] range value from parse err %s", err.Error())) } } if v.To != "" { to, err = time.ParseInLocation(format, v.To, timeZone) if err != nil { - return errors.New(errors.ErrorTypeIllegalArgumentException, "[date_range] range value to parse error "+err.Error()) + return errors.New(errors.ErrorTypeIllegalArgumentException, fmt.Sprintf("[date_range] range value to parse err %s", err.Error())) } } subreq.AddRange(aggregations.NewDateRange(from, to)) @@ -143,7 +143,7 @@ func Request(req zincaggregation.SearchAggregation, aggs map[string]meta.Aggrega default: return errors.New( errors.ErrorTypeParsingException, - fmt.Sprintf("[histogram] aggregation doesn't support values of type: [%s:[%v]]", agg.Histogram.Field, mappings.Properties[agg.Histogram.Field].Type), + fmt.Sprintf("[histogram] aggregation doesn't support values of type: [%s:[%s]]", agg.Histogram.Field, mappings.Properties[agg.Histogram.Field].Type), ) } if len(agg.Aggregations) > 0 { @@ -196,7 +196,7 @@ func Request(req zincaggregation.SearchAggregation, aggs map[string]meta.Aggrega if agg.DateHistogram.TimeZone != "" { timeZone, err = zutils.ParseTimeZone(agg.DateHistogram.TimeZone) if err != nil { - return errors.New(errors.ErrorTypeXContentParseException, fmt.Sprintf("[date_histogram] time_zone parse err %v", err)) + return errors.New(errors.ErrorTypeXContentParseException, fmt.Sprintf("[date_histogram] time_zone parse err %s", err.Error())) } } if agg.DateHistogram.Format == "" { @@ -218,7 +218,7 @@ func Request(req zincaggregation.SearchAggregation, aggs map[string]meta.Aggrega return errors.New( errors.ErrorTypeParsingException, fmt.Sprintf( - "[date_histogram] aggregation doesn't support values of type: [%s:[%v]]", + "[date_histogram] aggregation doesn't support values of type: [%s:[%s]]", agg.DateHistogram.Field, mappings.Properties[agg.DateHistogram.Field].Type, ), @@ -253,7 +253,7 @@ func Request(req zincaggregation.SearchAggregation, aggs map[string]meta.Aggrega if agg.AutoDateHistogram.TimeZone != "" { timeZone, err = zutils.ParseTimeZone(agg.AutoDateHistogram.TimeZone) if err != nil { - return errors.New(errors.ErrorTypeXContentParseException, fmt.Sprintf("[auto_date_histogram] time_zone parse err %v", err)) + return errors.New(errors.ErrorTypeXContentParseException, fmt.Sprintf("[auto_date_histogram] time_zone parse err %s", err.Error())) } } if agg.AutoDateHistogram.Format == "" { @@ -273,7 +273,7 @@ func Request(req zincaggregation.SearchAggregation, aggs map[string]meta.Aggrega return errors.New( errors.ErrorTypeParsingException, fmt.Sprintf( - "[auto_date_histogram] aggregation doesn't support values of type: [%s:[%v]]", + "[auto_date_histogram] aggregation doesn't support values of type: [%s:[%s]]", agg.AutoDateHistogram.Field, mappings.Properties[agg.AutoDateHistogram.Field].Type, ), diff --git a/pkg/uquery/v2/analysis/analyzer/regexp.go b/pkg/uquery/v2/analysis/analyzer/regexp.go index 556119cb6..f616cd288 100644 --- a/pkg/uquery/v2/analysis/analyzer/regexp.go +++ b/pkg/uquery/v2/analysis/analyzer/regexp.go @@ -26,7 +26,7 @@ func NewRegexpAnalyzer(options interface{}) (*analysis.Analyzer, error) { r, err := regexp.Compile(pattern) if err != nil { - return nil, errors.New(errors.ErrorTypeParsingException, fmt.Sprintf("[analyzer] regexp option [pattern] compile error: %v", err.Error())) + return nil, errors.New(errors.ErrorTypeParsingException, fmt.Sprintf("[analyzer] regexp option [pattern] compile error: %s", err.Error())) } ana := &analysis.Analyzer{Tokenizer: tokenizer.NewRegexpTokenizer(r)} diff --git a/pkg/uquery/v2/analysis/char/regexp.go b/pkg/uquery/v2/analysis/char/regexp.go index 6e024398c..aa8178b9e 100644 --- a/pkg/uquery/v2/analysis/char/regexp.go +++ b/pkg/uquery/v2/analysis/char/regexp.go @@ -19,7 +19,7 @@ func NewRegexpCharFilter(options interface{}) (analysis.CharFilter, error) { replacement, _ := zutils.GetStringFromMap(options, "replacement") r, err := regexp.Compile(pattern) if err != nil { - return nil, errors.New(errors.ErrorTypeParsingException, fmt.Sprintf("[char_filter] regexp option [pattern] compile error: %v", err.Error())) + return nil, errors.New(errors.ErrorTypeParsingException, fmt.Sprintf("[char_filter] regexp option [pattern] compile error: %s", err.Error())) } return char.NewRegexpCharFilter(r, []byte(replacement)), nil diff --git a/pkg/uquery/v2/analysis/token/regexp.go b/pkg/uquery/v2/analysis/token/regexp.go index d4d61b4c3..665c64e08 100644 --- a/pkg/uquery/v2/analysis/token/regexp.go +++ b/pkg/uquery/v2/analysis/token/regexp.go @@ -19,7 +19,7 @@ func NewRegexpTokenFilter(options interface{}) (analysis.TokenFilter, error) { replacement, _ := zutils.GetStringFromMap(options, "replacement") r, err := regexp.Compile(pattern) if err != nil { - return nil, errors.New(errors.ErrorTypeParsingException, fmt.Sprintf("[token_filter] regexp option [pattern] compile error: %v", err.Error())) + return nil, errors.New(errors.ErrorTypeParsingException, fmt.Sprintf("[token_filter] regexp option [pattern] compile error: %s", err.Error())) } return token.NewRegexpTokenFilter(r, []byte(replacement)), nil diff --git a/pkg/uquery/v2/analysis/tokenizer/exception.go b/pkg/uquery/v2/analysis/tokenizer/exception.go index 2114aa8fb..582b20bec 100644 --- a/pkg/uquery/v2/analysis/tokenizer/exception.go +++ b/pkg/uquery/v2/analysis/tokenizer/exception.go @@ -21,7 +21,7 @@ func NewExceptionTokenizer(options interface{}) (analysis.Tokenizer, error) { pattern := strings.Join(patterns, "|") r, err := regexp.Compile(pattern) if err != nil { - return nil, errors.New(errors.ErrorTypeParsingException, fmt.Sprintf("[tokenizer] exception option [patterns] compile error: %v", err.Error())) + return nil, errors.New(errors.ErrorTypeParsingException, fmt.Sprintf("[tokenizer] exception option [patterns] compile error: %s", err.Error())) } return tokenizer.NewExceptionsTokenizer(r, tokenizer.NewUnicodeTokenizer()), nil diff --git a/pkg/uquery/v2/analysis/tokenizer/regexp.go b/pkg/uquery/v2/analysis/tokenizer/regexp.go index ab140cae9..f955a2c80 100644 --- a/pkg/uquery/v2/analysis/tokenizer/regexp.go +++ b/pkg/uquery/v2/analysis/tokenizer/regexp.go @@ -18,7 +18,7 @@ func NewRegexpTokenizer(options interface{}) (analysis.Tokenizer, error) { r, err := regexp.Compile(pattern) if err != nil { - return nil, errors.New(errors.ErrorTypeParsingException, fmt.Sprintf("[tokenizer] regexp option [pattern] compile error: %v", err.Error())) + return nil, errors.New(errors.ErrorTypeParsingException, fmt.Sprintf("[tokenizer] regexp option [pattern] compile error: %s", err.Error())) } return tokenizer.NewRegexpTokenizer(r), nil diff --git a/pkg/uquery/v2/fields/fields.go b/pkg/uquery/v2/fields/fields.go index 6c5aa423c..23b50e2d8 100644 --- a/pkg/uquery/v2/fields/fields.go +++ b/pkg/uquery/v2/fields/fields.go @@ -1,11 +1,12 @@ package fields import ( - "encoding/json" "fmt" "strings" "time" + "github.com/goccy/go-json" + "github.com/zinclabs/zinc/pkg/errors" meta "github.com/zinclabs/zinc/pkg/meta/v2" ) diff --git a/pkg/uquery/v2/index/index.go b/pkg/uquery/v2/index/index.go index 85c7367f9..59f9613b7 100644 --- a/pkg/uquery/v2/index/index.go +++ b/pkg/uquery/v2/index/index.go @@ -1,10 +1,10 @@ package index import ( - "encoding/json" "fmt" "github.com/blugelabs/bluge/analysis" + "github.com/goccy/go-json" "github.com/zinclabs/zinc/pkg/errors" meta "github.com/zinclabs/zinc/pkg/meta/v2" diff --git a/pkg/uquery/v2/query/range.go b/pkg/uquery/v2/query/range.go index 9c902350b..6227aab3a 100644 --- a/pkg/uquery/v2/query/range.go +++ b/pkg/uquery/v2/query/range.go @@ -126,7 +126,7 @@ func RangeQueryTime(field string, query map[string]interface{}, mappings *meta.M if value.TimeZone != "" { timeZone, err = zutils.ParseTimeZone(value.TimeZone) if err != nil { - return nil, errors.New(errors.ErrorTypeXContentParseException, fmt.Sprintf("[range] %s time_zone parse err %v", field, err)) + return nil, errors.New(errors.ErrorTypeXContentParseException, fmt.Sprintf("[range] %s time_zone parse err %s", field, err.Error())) } } @@ -137,27 +137,27 @@ func RangeQueryTime(field string, query map[string]interface{}, mappings *meta.M if value.GT != nil && value.GT.(string) != "" { min, err = time.ParseInLocation(format, value.GT.(string), timeZone) if err != nil { - return nil, errors.New(errors.ErrorTypeXContentParseException, fmt.Sprintf("[range] %s range.gt format err %v", field, err)) + return nil, errors.New(errors.ErrorTypeXContentParseException, fmt.Sprintf("[range] %s range.gt format err %s", field, err.Error())) } } if value.GTE != nil && value.GTE.(string) != "" { minInclusive = true min, err = time.ParseInLocation(format, value.GTE.(string), timeZone) if err != nil { - return nil, errors.New(errors.ErrorTypeXContentParseException, fmt.Sprintf("[range] %s range.gte format err %v", field, err)) + return nil, errors.New(errors.ErrorTypeXContentParseException, fmt.Sprintf("[range] %s range.gte format err %s", field, err.Error())) } } if value.LT != nil && value.LT.(string) != "" { max, err = time.ParseInLocation(format, value.LT.(string), timeZone) if err != nil { - return nil, errors.New(errors.ErrorTypeXContentParseException, fmt.Sprintf("[range] %s range.lt format err %v", field, err)) + return nil, errors.New(errors.ErrorTypeXContentParseException, fmt.Sprintf("[range] %s range.lt format err %s", field, err.Error())) } } if value.LTE != nil && value.LTE.(string) != "" { maxInclusive = true max, err = time.ParseInLocation(format, value.LTE.(string), timeZone) if err != nil { - return nil, errors.New(errors.ErrorTypeXContentParseException, fmt.Sprintf("[range] %s range.lte format err %v", field, err)) + return nil, errors.New(errors.ErrorTypeXContentParseException, fmt.Sprintf("[range] %s range.lte format err %s", field, err.Error())) } } subq := bluge.NewDateRangeInclusiveQuery(min.UTC(), max.UTC(), minInclusive, maxInclusive).SetField(field) diff --git a/pkg/uquery/v2/source/source.go b/pkg/uquery/v2/source/source.go index da318a818..f41af12b6 100644 --- a/pkg/uquery/v2/source/source.go +++ b/pkg/uquery/v2/source/source.go @@ -1,9 +1,10 @@ package source import ( - "encoding/json" "strings" + "github.com/goccy/go-json" + "github.com/zinclabs/zinc/pkg/errors" meta "github.com/zinclabs/zinc/pkg/meta/v2" ) diff --git a/pkg/zutils/flatten/flatten.go b/pkg/zutils/flatten/flatten.go index 4b03cead3..c8833a3d4 100644 --- a/pkg/zutils/flatten/flatten.go +++ b/pkg/zutils/flatten/flatten.go @@ -27,14 +27,14 @@ func Unflatten(flat map[string]interface{}) (map[string]interface{}, error) { innerMap, ok := v.(map[string]interface{}) if !ok { - return nil, fmt.Errorf("key=%v is not an object", strings.Join(keyParts[0:i+1], ".")) + return nil, fmt.Errorf("key=%s is not an object", strings.Join(keyParts[0:i+1], ".")) } m = innerMap } leafKey := keyParts[len(keyParts)-1] if _, exists := m[leafKey]; exists { - return nil, fmt.Errorf("key=%v already exists", key) + return nil, fmt.Errorf("key=%s already exists", key) } m[keyParts[len(keyParts)-1]] = value }