Merge pull request #158 from zinclabs/time-index

fixed: panic search on memory data

go.mod

@@ -1,6 +1,6 @@
 module github.com/zinclabs/zinc
 
-go 1.17
+go 1.16
 
 require (
 	github.com/aws/aws-sdk-go-v2/config v1.11.0
@@ -28,80 +28,19 @@ require (
 )
 
 require (
-	github.com/RoaringBitmap/roaring v0.9.4 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.11.2 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.0.0 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.6.4 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.2 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.2 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.2 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.5.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.9.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.11.1 // indirect
-	github.com/aws/smithy-go v1.9.0 // indirect
-	github.com/axiomhq/hyperloglog v0.0.0-20191112132149-a4c4c47bc57f // indirect
-	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/bits-and-blooms/bitset v1.2.0 // indirect
-	github.com/blevesearch/go-porterstemmer v1.0.3 // indirect
-	github.com/blevesearch/mmap-go v1.0.3 // indirect
-	github.com/blevesearch/segment v0.9.0 // indirect
-	github.com/blevesearch/snowballstem v0.9.0 // indirect
-	github.com/blevesearch/vellum v1.0.7 // indirect
-	github.com/blugelabs/ice v0.2.0 // indirect
 	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
-	github.com/caio/go-tdigest v3.1.0+incompatible // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc // indirect
-	github.com/dustin/go-humanize v1.0.0 // indirect
-	github.com/gin-contrib/sse v0.1.0 // indirect
-	github.com/go-ole/go-ole v1.2.6 // indirect
-	github.com/go-playground/locales v0.13.0 // indirect
-	github.com/go-playground/universal-translator v0.17.0 // indirect
-	github.com/go-playground/validator/v10 v10.4.1 // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/uuid v1.3.0 // indirect
-	github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect
-	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/jtolds/gls v4.20.0+incompatible // indirect
-	github.com/klauspost/compress v1.15.2 // indirect
-	github.com/klauspost/cpuid v1.3.1 // indirect
-	github.com/leodido/go-urn v1.2.0 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
-	github.com/minio/md5-simd v1.1.0 // indirect
-	github.com/minio/sha256-simd v0.1.1 // indirect
-	github.com/mitchellh/go-homedir v1.1.0 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/mschoch/smat v0.2.0 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_golang v1.12.0 // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.32.1 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/rs/xid v1.3.0 // indirect
 	github.com/segmentio/backo-go v1.0.0 // indirect
-	github.com/sirupsen/logrus v1.8.1 // indirect
-	github.com/smartystreets/assertions v1.2.0 // indirect
 	github.com/tklauser/go-sysconf v0.3.9 // indirect
-	github.com/tklauser/numcpus v0.3.0 // indirect
-	github.com/ugorji/go/codec v1.1.7 // indirect
-	github.com/vcaesar/cedar v0.20.0 // indirect
 	github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
 	golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect
 	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
-	google.golang.org/protobuf v1.26.0 // indirect
-	gopkg.in/ini.v1 v1.57.0 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
 )
 
-replace github.com/blugelabs/bluge => github.com/zinclabs/bluge v0.1.9-0.20220518121859-2fc6f6827d84
+replace github.com/blugelabs/bluge => ../bluge
 
-replace github.com/blugelabs/ice => github.com/zinclabs/ice v0.2.1-0.20220515034015-5e1f80d8e605
+replace github.com/blugelabs/ice => github.com/zinclabs/ice v0.2.1-0.20220523154843-772e1ae38b48
+
+replace github.com/blugelabs/bluge_segment_api => github.com/zinclabs/bluge_segment_api v0.2.1-0.20220523030708-2e8f9721fa17

go.sum

@@ -101,8 +101,6 @@ github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD
 github.com/blevesearch/snowballstem v0.9.0/go.mod h1:PivSj3JMc8WuaFkTSRDW2SlrulNWPl4ABg1tC/hlgLs=
 github.com/blevesearch/vellum v1.0.7 h1:+vn8rfyCRHxKVRgDLeR0FAXej2+6mEb5Q15aQE/XESQ=
 github.com/blevesearch/vellum v1.0.7/go.mod h1:doBZpmRhwTsASB4QdUZANlJvqVAUdUyX0ZK7QJCTeBE=
-github.com/blugelabs/bluge_segment_api v0.2.0 h1:cCX1Y2y8v0LZ7+EEJ6gH7dW6TtVTW4RhG0vp3R+N2Lo=
-github.com/blugelabs/bluge_segment_api v0.2.0/go.mod h1:95XA+ZXfRj/IXADm7gZ+iTcWOJPg5jQTY1EReIzl3LA=
 github.com/blugelabs/query_string v0.3.0 h1:/2XMd/3A0lIo33STXMUon4khLcTTybrAcyC0mBLmeA8=
 github.com/blugelabs/query_string v0.3.0/go.mod h1:H0YFWhYAf8/xcv1zoswoUC8kM/fE9L/KEfsgySsnhfs=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
@@ -463,10 +461,10 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
 github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
-github.com/zinclabs/bluge v0.1.9-0.20220518121859-2fc6f6827d84 h1:JyQFYI+9tZOZGlnv23LERe3XOr18szBsskCfZfNWtdM=
-github.com/zinclabs/bluge v0.1.9-0.20220518121859-2fc6f6827d84/go.mod h1:krmos0IIZkhCVgytnsRAeKFDs/BIvLOubRquJKlZFdo=
-github.com/zinclabs/ice v0.2.1-0.20220515034015-5e1f80d8e605 h1:mXhz8ObmUjiyuMkExk9yzFRJJUu/ijokyNWcTy6RKhE=
-github.com/zinclabs/ice v0.2.1-0.20220515034015-5e1f80d8e605/go.mod h1:5zqcEYZJIlqtnPQwAl+fXfWVErh07MwYWi6TriGyMV4=
+github.com/zinclabs/bluge_segment_api v0.2.1-0.20220523030708-2e8f9721fa17 h1:6nZipqKn4MY+Jb3uHtzKvQCbHCWJWVbLy7LwoA9rZq0=
+github.com/zinclabs/bluge_segment_api v0.2.1-0.20220523030708-2e8f9721fa17/go.mod h1:w+BFZs3+UsRmNSvE/Wxl7TSsZQBcso7yEwyYQ4Ovw+M=
+github.com/zinclabs/ice v0.2.1-0.20220523154843-772e1ae38b48 h1:Ze5gzqhD2rhLsy+OUjr00Q6ZseJwcq8/Ku5JSNDnZe0=
+github.com/zinclabs/ice v0.2.1-0.20220523154843-772e1ae38b48/go.mod h1:b7FG0NtjT+8iOsP57UWzG4V4WlNVrl0VG2YnOk1o82k=
 github.com/zsais/go-gin-prometheus v0.0.0-20200217150448-2199a42d96c1 h1:wEkwpwzLhU3VEZRV8kS3eeYnyps4udKe14RQOmbjsZI=
 github.com/zsais/go-gin-prometheus v0.0.0-20200217150448-2199a42d96c1/go.mod h1:Slirjzuz8uM8Cw0jmPNqbneoqcUtY2GGjn2bEd4NRLY=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=

multiarch.sh

@@ -4,6 +4,6 @@ aws ecr-public get-login-password --region us-east-1 | docker login --username A
 
 # docker buildx build --push --platform linux/arm/v7,linux/arm64/v8,linux/amd64 --tag public.ecr.aws/zinclabs/zinc:v0.1.3-s3test .
 
-docker buildx build --push --platform linux/amd64 --tag public.ecr.aws/zinclabs/zinc:v0.1.3-s3test .
+docker buildx build --push --platform linux/amd64 --tag public.ecr.aws/zinclabs/zinc:test .
 
 

pkg/core/index.go

@@ -56,25 +56,25 @@ func (index *Index) BuildBlugeDocumentFromJSON(docID string, doc map[string]inte
 			continue
 		}
 
-		if _, ok := mappings.Properties[key]; !ok {
+		if _, ok := mappings.GetProperty(key); !ok {
 			// try to find the type of the value and use it to define default mapping
 			switch value.(type) {
 			case string:
-				mappings.Properties[key] = meta.NewProperty("text")
+				mappings.SetProperty(key, meta.NewProperty("text"))
 			case float64:
-				mappings.Properties[key] = meta.NewProperty("numeric")
+				mappings.SetProperty(key, meta.NewProperty("numeric"))
 			case bool:
-				mappings.Properties[key] = meta.NewProperty("bool")
+				mappings.SetProperty(key, meta.NewProperty("bool"))
 			case []interface{}:
 				if v, ok := value.([]interface{}); ok {
 					for _, vv := range v {
 						switch vv.(type) {
 						case string:
-							mappings.Properties[key] = meta.NewProperty("text")
+							mappings.SetProperty(key, meta.NewProperty("text"))
 						case float64:
-							mappings.Properties[key] = meta.NewProperty("numeric")
+							mappings.SetProperty(key, meta.NewProperty("numeric"))
 						case bool:
-							mappings.Properties[key] = meta.NewProperty("bool")
+							mappings.SetProperty(key, meta.NewProperty("bool"))
 						}
 						break
 					}
@@ -84,7 +84,7 @@ func (index *Index) BuildBlugeDocumentFromJSON(docID string, doc map[string]inte
 			mappingsNeedsUpdate = true
 		}
 
-		if !mappings.Properties[key].Index {
+		if prop, ok := mappings.GetProperty(key); ok && !prop.Index {
 			continue // not index, skip
 		}
 
@@ -128,14 +128,18 @@ func (index *Index) BuildBlugeDocumentFromJSON(docID string, doc map[string]inte
 	bdoc.AddField(bluge.NewDateTimeField("@timestamp", timestamp).StoreValue().Sortable().Aggregatable())
 	bdoc.AddField(bluge.NewStoredOnlyField("_index", []byte(index.Name)))
 	bdoc.AddField(bluge.NewStoredOnlyField("_source", docByteVal))
-	bdoc.AddField(bluge.NewCompositeFieldExcluding("_all", []string{"_index", "_id", "_source", "@timestamp"}))
+	bdoc.AddField(bluge.NewCompositeFieldExcluding("_all", []string{"_id", "_index", "_source", "@timestamp"}))
+
+	// test for add time index
+	bdoc.SetTimestamp(timestamp.UnixNano())
 
 	return bdoc, nil
 }
 
 func (index *Index) buildField(mappings *meta.Mappings, bdoc *bluge.Document, key string, value interface{}) error {
 	var field *bluge.TermField
-	switch mappings.Properties[key].Type {
+	prop, _ := mappings.GetProperty(key)
+	switch prop.Type {
 	case "text":
 		v, ok := value.(string)
 		if !ok {
@@ -172,8 +176,8 @@ func (index *Index) buildField(mappings *meta.Mappings, bdoc *bluge.Document, ke
 		switch v := value.(type) {
 		case string:
 			format := time.RFC3339
-			if mappings.Properties[key].Format != "" {
-				format = mappings.Properties[key].Format
+			if prop.Format != "" {
+				format = prop.Format
 			}
 			var tim time.Time
 			var err error
@@ -192,19 +196,19 @@ func (index *Index) buildField(mappings *meta.Mappings, bdoc *bluge.Document, ke
 			}
 		}
 	}
-	if mappings.Properties[key].Store {
+	if prop.Store {
 		field.StoreValue()
 	}
-	if mappings.Properties[key].Sortable {
+	if prop.Sortable {
 		field.Sortable()
 	}
-	if mappings.Properties[key].Aggregatable {
+	if prop.Aggregatable {
 		field.Aggregatable()
 	}
-	if mappings.Properties[key].Highlightable {
+	if prop.Highlightable {
 		field.HighlightMatches()
 	}
-	if mappings.Properties[key].TermPositions {
+	if prop.TermPositions {
 		field.SearchTermPositions()
 	}
 	bdoc.AddField(field)
@@ -254,22 +258,23 @@ func (index *Index) SetAnalyzers(analyzers map[string]*analysis.Analyzer) error
 }
 
 func (index *Index) SetMappings(mappings *meta.Mappings) error {
-	if mappings == nil || len(mappings.Properties) == 0 {
+	if mappings == nil || mappings.Len() == 0 {
 		return nil
 	}
 
 	// custom analyzer just for text field
-	for _, prop := range mappings.Properties {
+	for field, prop := range mappings.ListProperty() {
 		if prop.Type != "text" {
 			prop.Analyzer = ""
 			prop.SearchAnalyzer = ""
+			mappings.SetProperty(field, prop)
 		}
 	}
 
-	mappings.Properties["_id"] = meta.NewProperty("keyword")
+	mappings.SetProperty("_id", meta.NewProperty("keyword"))
 
 	// @timestamp need date_range/date_histogram aggregation, and mappings used for type check in aggregation
-	mappings.Properties["@timestamp"] = meta.NewProperty("date")
+	mappings.SetProperty("@timestamp", meta.NewProperty("date"))
 
 	// update in the cache
 	index.CachedMappings = mappings

pkg/core/index_test.go

@@ -104,11 +104,11 @@ func TestIndex_BuildBlugeDocumentFromJSON(t *testing.T) {
 				},
 			},
 			init: func() {
-				index.CachedMappings.Properties["time"] = meta.Property{
+				index.CachedMappings.SetProperty("time", meta.Property{
 					Type:   "time",
 					Index:  true,
 					Format: "2006-01-02 15:04:05.000",
-				}
+				})
 			},
 			want:    &bluge.Document{},
 			wantErr: false,
@@ -126,17 +126,17 @@ func TestIndex_BuildBlugeDocumentFromJSON(t *testing.T) {
 				},
 			},
 			init: func() {
-				index.CachedMappings.Properties["id"] = meta.Property{
+				index.CachedMappings.SetProperty("id", meta.Property{
 					Type:          "keyword",
 					Index:         true,
 					Store:         true,
 					Highlightable: true,
-				}
-				index.CachedMappings.Properties["name"] = meta.Property{
+				})
+				index.CachedMappings.SetProperty("name", meta.Property{
 					Type:     "text",
 					Index:    true,
 					Analyzer: "analyzer_1",
-				}
+				})
 				index.CachedAnalyzers["analyzer_1"] = analyzer.NewStandardAnalyzer()
 			},
 			want:    &bluge.Document{},
@@ -228,7 +228,7 @@ func TestIndex_BuildBlugeDocumentFromJSON(t *testing.T) {
 
 		err = StoreIndex(index)
 		assert.NoError(t, err)
-		index.CachedMappings.Properties["time"] = meta.NewProperty("time")
+		index.CachedMappings.SetProperty("time", meta.NewProperty("date"))
 	})
 
 	for _, tt := range tests {

pkg/core/search_test.go

@@ -290,12 +290,12 @@ func TestIndex_Search(t *testing.T) {
 			if (index.CachedMappings) == nil {
 				index.CachedMappings = meta.NewMappings()
 			}
-			index.CachedMappings.Properties["address.city"] = meta.Property{
+			index.CachedMappings.SetProperty("address.city", meta.Property{
 				Type:          "text",
 				Index:         true,
 				Store:         true,
 				Highlightable: true,
-			}
+			})
 
 			for _, d := range tt.data {
 				rand.Seed(time.Now().UnixNano())

pkg/handlers/index/analyzer.go

@@ -46,8 +46,8 @@ func Analyze(c *gin.Context) {
 			return
 		}
 		if query.Filed != "" && query.Analyzer == "" {
-			if index.CachedMappings != nil && index.CachedMappings.Properties != nil {
-				if prop, ok := index.CachedMappings.Properties[query.Filed]; ok {
+			if index.CachedMappings != nil && index.CachedMappings.Len() > 0 {
+				if prop, ok := index.CachedMappings.GetProperty(query.Filed); ok {
 					if query.Analyzer == "" && prop.SearchAnalyzer != "" {
 						query.Analyzer = prop.SearchAnalyzer
 					}

pkg/handlers/index/mapping.go

@@ -58,7 +58,7 @@ func SetMapping(c *gin.Context) {
 	index, exists := core.GetIndex(indexName)
 	if exists {
 		// check if mapping is empty
-		if index.CachedMappings != nil && len(index.CachedMappings.Properties) > 0 {
+		if index.CachedMappings != nil && index.CachedMappings.Len() > 0 {
 			c.JSON(http.StatusBadRequest, gin.H{"error": "index [" + indexName + "] already exists"})
 			return
 		}
@@ -77,7 +77,7 @@ func SetMapping(c *gin.Context) {
 	}
 
 	// update mappings
-	if mappings != nil && len(mappings.Properties) > 0 {
+	if mappings != nil && mappings.Len() > 0 {
 		_ = index.SetMappings(mappings)
 	}
 

pkg/meta/mappings.go

@@ -15,8 +15,11 @@
 
 package meta
 
+import "sync"
+
 type Mappings struct {
 	Properties map[string]Property `json:"properties,omitempty"`
+	lock       sync.RWMutex
 }
 
 type Property struct {
@@ -59,3 +62,33 @@ func NewProperty(typ string) Property {
 
 	return p
 }
+
+func (t *Mappings) Len() int {
+	t.lock.RLock()
+	n := len(t.Properties)
+	t.lock.RUnlock()
+	return n
+}
+
+func (t *Mappings) SetProperty(field string, prop Property) {
+	t.lock.Lock()
+	t.Properties[field] = prop
+	t.lock.Unlock()
+}
+
+func (t *Mappings) GetProperty(field string) (Property, bool) {
+	t.lock.RLock()
+	prop, ok := t.Properties[field]
+	t.lock.RUnlock()
+	return prop, ok
+}
+
+func (t *Mappings) ListProperty() map[string]Property {
+	m := make(map[string]Property)
+	t.lock.RLock()
+	for k, v := range t.Properties {
+		m[k] = v
+	}
+	t.lock.RUnlock()
+	return m
+}