fix: reset search query when select index changed (#121)

* fix: reset search query when select index changed * fix: reset search query when select index changed * fix: compatible es index template * fix: compatible es index template * feat: change behavior of search fields * feat: allow multiple values for a field
zincsearch · Mar 30, 2022 · 2dcceb2 · 2dcceb2
1 parent b330bbf
commit 2dcceb2
Show file tree

Hide file tree

Showing 15 changed files with 410 additions and 153 deletions.
diff --git a/pkg/core/index.go b/pkg/core/index.go
@@ -6,19 +6,18 @@ import (
 	"fmt"
 	"math"
 	"path/filepath"
-	"reflect"
 	"strconv"
 	"sync/atomic"
 	"time"
 
 	"github.com/blugelabs/bluge"
 	"github.com/blugelabs/bluge/analysis"
-	"github.com/jeremywohl/flatten"
 	"github.com/rs/zerolog/log"
 
 	meta "github.com/prabhatsharma/zinc/pkg/meta/v2"
 	zincanalysis "github.com/prabhatsharma/zinc/pkg/uquery/v2/analysis"
 	"github.com/prabhatsharma/zinc/pkg/zutils"
+	"github.com/prabhatsharma/zinc/pkg/zutils/flatten"
 )
 
 // BuildBlugeDocumentFromJSON returns the bluge document for the json document. It also updates the mapping for the fields if not found.
@@ -35,28 +34,36 @@ func (index *Index) BuildBlugeDocumentFromJSON(docID string, doc *map[string]int
 
 	// Create a new bluge document
 	bdoc := bluge.NewDocument(docID)
-	flatDoc, _ := flatten.Flatten(*doc, "", flatten.DotStyle)
+	flatDoc, _ := flatten.Flatten(*doc, "")
 	// Iterate through each field and add it to the bluge document
 	for key, value := range flatDoc {
 		if value == nil {
 			continue
 		}
 
 		if _, ok := mappings.Properties[key]; !ok {
-			// Use reflection to find the type of the value.
-			// Bluge requires the field type to be specified.
-			v := reflect.ValueOf(value)
-
 			// try to find the type of the value and use it to define default mapping
-			switch v.Type().String() {
-			case "string":
+			switch value.(type) {
+			case string:
 				mappings.Properties[key] = meta.NewProperty("text")
-			case "float64":
+			case float64:
 				mappings.Properties[key] = meta.NewProperty("numeric")
-			case "bool":
+			case bool:
 				mappings.Properties[key] = meta.NewProperty("bool")
-			case "time.Time":
-				mappings.Properties[key] = meta.NewProperty("time")
+			case []interface{}:
+				if v, ok := value.([]interface{}); ok {
+					for _, vv := range v {
+						switch vv.(type) {
+						case string:
+							mappings.Properties[key] = meta.NewProperty("text")
+						case float64:
+							mappings.Properties[key] = meta.NewProperty("numeric")
+						case bool:
+							mappings.Properties[key] = meta.NewProperty("bool")
+						}
+						break
+					}
+				}
 			}
 
 			mappingsNeedsUpdate = true
@@ -66,53 +73,18 @@ func (index *Index) BuildBlugeDocumentFromJSON(docID string, doc *map[string]int
 			continue // not index, skip
 		}
 
-		var field *bluge.TermField
-		switch mappings.Properties[key].Type {
-		case "text":
-			field = bluge.NewTextField(key, value.(string)).SearchTermPositions()
-			fieldAnalyzer, _ := zincanalysis.QueryAnalyzerForField(index.CachedAnalyzers, index.CachedMappings, key)
-			if fieldAnalyzer != nil {
-				field.WithAnalyzer(fieldAnalyzer)
-			}
-		case "numeric":
-			field = bluge.NewNumericField(key, value.(float64))
-		case "keyword":
-			// compatible verion <= v0.1.4
-			if v, ok := value.(bool); ok {
-				field = bluge.NewKeywordField(key, strconv.FormatBool(v))
-			} else if v, ok := value.(string); ok {
-				field = bluge.NewKeywordField(key, v)
-			} else {
-				return nil, fmt.Errorf("keyword type only support text")
-			}
-		case "bool": // found using existing index mapping
-			value := value.(bool)
-			field = bluge.NewKeywordField(key, strconv.FormatBool(value))
-		case "time":
-			format := time.RFC3339
-			if mappings.Properties[key].Format != "" {
-				format = mappings.Properties[key].Format
+		switch v := value.(type) {
+		case []interface{}:
+			for _, v := range v {
+				if err := index.buildField(mappings, bdoc, key, v); err != nil {
+					return nil, err
+				}
 			}
-			tim, err := time.Parse(format, value.(string))
-			if err != nil {
+		default:
+			if err := index.buildField(mappings, bdoc, key, v); err != nil {
 				return nil, err
 			}
-			field = bluge.NewDateTimeField(key, tim)
-		}
-
-		if mappings.Properties[key].Store {
-			field.StoreValue()
 		}
-		if mappings.Properties[key].Sortable {
-			field.Sortable()
-		}
-		if mappings.Properties[key].Aggregatable {
-			field.Aggregatable()
-		}
-		if mappings.Properties[key].Highlightable {
-			field.HighlightMatches()
-		}
-		bdoc.AddField(field)
 	}
 
 	if mappingsNeedsUpdate {
@@ -137,6 +109,58 @@ func (index *Index) BuildBlugeDocumentFromJSON(docID string, doc *map[string]int
 	return bdoc, nil
 }
 
+func (index *Index) buildField(mappings *meta.Mappings, bdoc *bluge.Document, key string, value interface{}) error {
+	var field *bluge.TermField
+	switch mappings.Properties[key].Type {
+	case "text":
+		field = bluge.NewTextField(key, value.(string)).SearchTermPositions()
+		fieldAnalyzer, _ := zincanalysis.QueryAnalyzerForField(index.CachedAnalyzers, index.CachedMappings, key)
+		if fieldAnalyzer != nil {
+			field.WithAnalyzer(fieldAnalyzer)
+		}
+	case "numeric":
+		field = bluge.NewNumericField(key, value.(float64))
+	case "keyword":
+		// compatible verion <= v0.1.4
+		if v, ok := value.(bool); ok {
+			field = bluge.NewKeywordField(key, strconv.FormatBool(v))
+		} else if v, ok := value.(string); ok {
+			field = bluge.NewKeywordField(key, v)
+		} else {
+			return fmt.Errorf("keyword type only support text")
+		}
+	case "bool": // found using existing index mapping
+		value := value.(bool)
+		field = bluge.NewKeywordField(key, strconv.FormatBool(value))
+	case "time":
+		format := time.RFC3339
+		if mappings.Properties[key].Format != "" {
+			format = mappings.Properties[key].Format
+		}
+		tim, err := time.Parse(format, value.(string))
+		if err != nil {
+			return err
+		}
+		field = bluge.NewDateTimeField(key, tim)
+	}
+
+	if mappings.Properties[key].Store {
+		field.StoreValue()
+	}
+	if mappings.Properties[key].Sortable {
+		field.Sortable()
+	}
+	if mappings.Properties[key].Aggregatable {
+		field.Aggregatable()
+	}
+	if mappings.Properties[key].Highlightable {
+		field.HighlightMatches()
+	}
+	bdoc.AddField(field)
+
+	return nil
+}
+
 func (index *Index) UseTemplate() error {
 	template, err := UseTemplate(index.Name)
 	if err != nil {

diff --git a/pkg/handlers/updatedocument.go b/pkg/handlers/updatedocument.go
@@ -48,7 +48,6 @@ func UpdateDocument(c *gin.Context) {
 		core.StoreIndex(index)
 	}
 
-	// doc, _ = flatten.Flatten(doc, "", flatten.DotStyle)
 	err = index.UpdateDocument(docID, &doc, mintedID)
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, err)

diff --git a/pkg/meta/v1/elasticsearch.go b/pkg/meta/v1/elasticsearch.go
@@ -1,14 +1,33 @@
 package v1
 
-import "strings"
+import (
+	"regexp"
+	"strings"
 
-func NewESInfo() *ESInfo {
+	"github.com/gin-gonic/gin"
+
+	"github.com/prabhatsharma/zinc/pkg/zutils"
+)
+
+func NewESInfo(c *gin.Context) *ESInfo {
+	version := strings.TrimLeft(Version, "v")
+	userAgent := c.Request.UserAgent()
+	if strings.Contains(userAgent, "Elastic") {
+		reg := regexp.MustCompile(`([0-9]+\.[0-9]+\.[0-9]+)`)
+		matches := reg.FindAllString(userAgent, 1)
+		if len(matches) > 0 {
+			version = matches[0]
+		}
+	}
+	if v := strings.ToUpper(zutils.GetEnv("ZINC_PLUGIN_ES_VERSION", "")); v != "" {
+		version = v
+	}
 	return &ESInfo{
 		Name:        "zinc",
 		ClusterName: "N/A",
 		ClusterUUID: "N/A",
 		Version: ESInfoVersion{
-			Number:                    strings.TrimLeft(Version, "v"),
+			Number:                    version,
 			BuildFlavor:               "default",
 			BuildHash:                 CommitHash,
 			BuildDate:                 BuildDate,
@@ -21,13 +40,13 @@ func NewESInfo() *ESInfo {
 	}
 }
 
-func NewESLicense() *ESLicense {
+func NewESLicense(_ *gin.Context) *ESLicense {
 	return &ESLicense{
 		Status: "active",
 	}
 }
 
-func NewESXPack() *ESXPack {
+func NewESXPack(_ *gin.Context) *ESXPack {
 	return &ESXPack{
 		Build:    make(map[string]bool),
 		Features: make(map[string]bool),

diff --git a/pkg/routes/routes.go b/pkg/routes/routes.go
@@ -96,13 +96,13 @@ func SetRoutes(r *gin.Engine) {
 	 */
 
 	r.GET("/es/", func(c *gin.Context) {
-		c.JSON(http.StatusOK, v1.NewESInfo())
+		c.JSON(http.StatusOK, v1.NewESInfo(c))
 	})
 	r.GET("/es/_license", func(c *gin.Context) {
-		c.JSON(http.StatusOK, v1.NewESLicense())
+		c.JSON(http.StatusOK, v1.NewESLicense(c))
 	})
 	r.GET("/es/_xpack", func(c *gin.Context) {
-		c.JSON(http.StatusOK, v1.NewESXPack())
+		c.JSON(http.StatusOK, v1.NewESXPack(c))
 	})
 
 	r.POST("/es/_search", auth.ZincAuthMiddleware, handlersV2.SearchIndex)

diff --git a/pkg/uquery/v2/mappings/mappings.go b/pkg/uquery/v2/mappings/mappings.go
@@ -32,6 +32,19 @@ func Request(analyzers map[string]*analysis.Analyzer, data map[string]interface{
 		if !ok {
 			return nil, errors.New(errors.ErrorTypeParsingException, fmt.Sprintf("[mappings] properties [%s] should be an object", field))
 		}
+		if v, ok := prop["properties"]; ok {
+			if _, ok := v.(map[string]interface{}); !ok {
+				return nil, errors.New(errors.ErrorTypeParsingException, fmt.Sprintf("[mappings] properties [%s] should be an object", field))
+			}
+			if subMappings, err := Request(analyzers, prop); err == nil {
+				for k, v := range subMappings.Properties {
+					mappings.Properties[field+"."+k] = v
+				}
+			} else {
+				return nil, err
+			}
+			continue
+		}
 		propType, ok := prop["type"]
 		if !ok {
 			return nil, errors.New(errors.ErrorTypeParsingException, fmt.Sprintf("[mappings] properties [%s] should be exists", "type"))
@@ -46,13 +59,17 @@ func Request(analyzers map[string]*analysis.Analyzer, data map[string]interface{
 		switch propTypeStr {
 		case "text", "keyword", "numeric", "bool", "time":
 			newProp = meta.NewProperty(propTypeStr)
-		case "integer", "double", "long":
+		case "constant_keyword":
+			newProp = meta.NewProperty("keyword")
+		case "match_only_text":
+			newProp = meta.NewProperty("text")
+		case "integer", "double", "long", "short", "int", "float":
 			newProp = meta.NewProperty("numeric")
 		case "boolean":
 			newProp = meta.NewProperty("bool")
 		case "date", "datetime":
 			newProp = meta.NewProperty("time")
-		case "flattened", "object", "match_only_text":
+		case "flattened", "object", "nested", "wildcard", "byte", "alias", "geo_point", "ip", "ip_range", "scaled_float":
 			// ignore
 		default:
 			return nil, errors.New(errors.ErrorTypeXContentParseException, fmt.Sprintf("[mappings] properties [%s] doesn't support type [%s]", field, propTypeStr))

diff --git a/pkg/uquery/v2/template/template.go b/pkg/uquery/v2/template/template.go
@@ -23,7 +23,7 @@ func Request(data map[string]interface{}) (*meta.Template, error) {
 	for k, v := range data {
 		k = strings.ToLower(k)
 		switch k {
-		case "name":
+		case "name", "data_stream":
 			// ignore
 		case "index_patterns":
 			patterns, ok := v.([]interface{})

diff --git a/pkg/zutils/flatten.go b/pkg/zutils/flatten.go