MB-58034: Custom Date Time parsers not being applied on query object

mapping/document.go

@@ -39,11 +39,12 @@ import (
 // are used.  To disable this automatic handling, set
 // Dynamic to false.
 type DocumentMapping struct {
-	Enabled         bool                        `json:"enabled"`
-	Dynamic         bool                        `json:"dynamic"`
-	Properties      map[string]*DocumentMapping `json:"properties,omitempty"`
-	Fields          []*FieldMapping             `json:"fields,omitempty"`
-	DefaultAnalyzer string                      `json:"default_analyzer,omitempty"`
+	Enabled               bool                        `json:"enabled"`
+	Dynamic               bool                        `json:"dynamic"`
+	Properties            map[string]*DocumentMapping `json:"properties,omitempty"`
+	Fields                []*FieldMapping             `json:"fields,omitempty"`
+	DefaultAnalyzer       string                      `json:"default_analyzer,omitempty"`
+	DefaultDateTimeParser string                      `json:"default_date_time_parser,omitempty"`
 
 	// StructTagKey overrides "json" when looking for field names in struct tags
 	StructTagKey string `json:"struct_tag_key,omitempty"`
@@ -96,6 +97,14 @@ func (dm *DocumentMapping) analyzerNameForPath(path string) string {
 	return ""
 }
 
+func (dm *DocumentMapping) dateTimeParserForPath(path string) string {
+	field := dm.fieldDescribedByPath(path)
+	if field != nil {
+		return field.DateFormat
+	}
+	return ""
+}
+
 func (dm *DocumentMapping) fieldDescribedByPath(path string) *FieldMapping {
 	pathElements := decodePath(path)
 	if len(pathElements) > 1 {
@@ -265,6 +274,11 @@ func (dm *DocumentMapping) UnmarshalJSON(data []byte) error {
 			if err != nil {
 				return err
 			}
+		case "default_datetime_parser":
+			err := json.Unmarshal(v, &dm.DefaultDateTimeParser)
+			if err != nil {
+				return err
+			}
 		case "properties":
 			err := json.Unmarshal(v, &dm.Properties)
 			if err != nil {
@@ -308,6 +322,22 @@ func (dm *DocumentMapping) defaultAnalyzerName(path []string) string {
 	return rv
 }
 
+func (dm *DocumentMapping) defaultDateTimeParser(path []string) string {
+	current := dm
+	rv := current.DefaultDateTimeParser
+	for _, pathElement := range path {
+		var ok bool
+		current, ok = current.Properties[pathElement]
+		if !ok {
+			break
+		}
+		if current.DefaultDateTimeParser != "" {
+			rv = current.DefaultDateTimeParser
+		}
+	}
+	return rv
+}
+
 func (dm *DocumentMapping) walkDocument(data interface{}, path []string, indexes []uint64, context *walkContext) {
 	// allow default "json" tag to be overridden
 	structTagKey := dm.StructTagKey

mapping/index.go

@@ -17,6 +17,7 @@ package mapping
 import (
 	"encoding/json"
 	"fmt"
+
 	index "github.com/blevesearch/bleve_index_api"
 
 	"github.com/blevesearch/bleve/v2/analysis"
@@ -417,20 +418,43 @@ func (im *IndexMappingImpl) DateTimeParserNamed(name string) analysis.DateTimePa
 	return dateTimeParser
 }
 
-func (im *IndexMappingImpl) datetimeParserNameForPath(path string) string {
-
+func (im *IndexMappingImpl) DatetimeParserNameForPath(path string) string {
 	// first we look for explicit mapping on the field
 	for _, docMapping := range im.TypeMapping {
-		pathMapping, _ := docMapping.documentMappingForPath(path)
-		if pathMapping != nil {
-			if len(pathMapping.Fields) > 0 {
-				if pathMapping.Fields[0].Analyzer != "" {
-					return pathMapping.Fields[0].Analyzer
-				}
+		dateTimeParser := docMapping.dateTimeParserForPath(path)
+		if dateTimeParser != "" {
+			return dateTimeParser
+		}
+	}
+
+	// now try the default mapping
+	pathMapping, _ := im.DefaultMapping.documentMappingForPath(path)
+	if pathMapping != nil {
+		if len(pathMapping.Fields) > 0 {
+			if pathMapping.Fields[0].DateFormat != "" {
+				return pathMapping.Fields[0].DateFormat
 			}
 		}
 	}
 
+	// next we will try default date-time parsers for the path
+	pathDecoded := decodePath(path)
+	for _, docMapping := range im.TypeMapping {
+		if docMapping.Enabled {
+			rv := docMapping.defaultDateTimeParser(pathDecoded)
+			if rv != "" {
+				return rv
+			}
+		}
+	}
+	// now the default date-time parser for the default mapping
+	if im.DefaultMapping.Enabled {
+		rv := im.DefaultMapping.defaultDateTimeParser(pathDecoded)
+		if rv != "" {
+			return rv
+		}
+	}
+
 	return im.DefaultDateTimeParser
 }
 

mapping/mapping.go

@@ -49,6 +49,7 @@ type IndexMapping interface {
 	MapDocument(doc *document.Document, data interface{}) error
 	Validate() error
 
+	DatetimeParserNameForPath(path string) string
 	DateTimeParserNamed(name string) analysis.DateTimeParser
 
 	DefaultSearchField() string

search/query/date_range.go

@@ -30,10 +30,10 @@ import (
 	index "github.com/blevesearch/bleve_index_api"
 )
 
-// QueryDateTimeParser controls the default query date time parser
+// QueryDateTimeParser controls the default query date time parser.
 var QueryDateTimeParser = optional.Name
 
-// QueryDateTimeFormat controls the format when Marshaling to JSON
+// QueryDateTimeFormat controls the format when Marshaling to JSON.
 var QueryDateTimeFormat = time.RFC3339
 
 var cache = registry.NewCache()
@@ -91,6 +91,82 @@ type DateRangeQuery struct {
 	InclusiveEnd   *bool          `json:"inclusive_end,omitempty"`
 	FieldVal       string         `json:"field,omitempty"`
 	BoostVal       *Boost         `json:"boost,omitempty"`
+	InheritParser  bool           `json:"inherit_parser"`
+	RawStart       string         `json:"raw_start,omitempty"`
+	RawEnd         string         `json:"raw_end,omitempty"`
+}
+
+// UnmarshalJSON offers custom unmarshaling
+func (q *DateRangeQuery) UnmarshalJSON(data []byte) error {
+	var tmp map[string]json.RawMessage
+	err := json.Unmarshal(data, &tmp)
+	if err != nil {
+		return err
+	}
+	// set defaults
+	q.InheritParser = false
+
+	for k, v := range tmp {
+		switch k {
+		case "inclusive_start":
+			err := json.Unmarshal(v, &q.InclusiveStart)
+			if err != nil {
+				return err
+			}
+		case "inclusive_end":
+			err := json.Unmarshal(v, &q.InclusiveEnd)
+			if err != nil {
+				return err
+			}
+		case "field":
+			err := json.Unmarshal(v, &q.FieldVal)
+			if err != nil {
+				return err
+			}
+		case "boost":
+			err := json.Unmarshal(v, &q.BoostVal)
+			if err != nil {
+				return err
+			}
+		case "inherit_parser":
+			err := json.Unmarshal(v, &q.InheritParser)
+			if err != nil {
+				return err
+			}
+
+		}
+	}
+	if tmp["start"] != nil {
+		if q.InheritParser {
+			// inherit parser from index mapping
+			err := json.Unmarshal(tmp["start"], &q.RawStart)
+			if err != nil {
+				return err
+			}
+		} else {
+			// use QueryDateTimeParser
+			err := json.Unmarshal(tmp["start"], &q.Start)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	if tmp["end"] != nil {
+		if q.InheritParser {
+			// inherit parser from index mapping
+			err := json.Unmarshal(tmp["end"], &q.RawEnd)
+			if err != nil {
+				return err
+			}
+		} else {
+			// use QueryDateTimeParser
+			err := json.Unmarshal(tmp["end"], &q.End)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
 }
 
 // NewDateRangeQuery creates a new Query for ranges
@@ -117,6 +193,32 @@ func NewDateRangeInclusiveQuery(start, end time.Time, startInclusive, endInclusi
 	}
 }
 
+// NewRawDateRangeQuery creates a new Query for ranges
+// of date values.
+// Date strings are not parsed, and are used as is.
+// Either, but not both endpoints can be nil.
+// Used when start and end must be parsed by deriving
+// the parser from the index mapping for the queried field.
+func NewDateRangeRawQuery(start, end string) *DateRangeQuery {
+	return NewDateRangeRawInclusiveQuery(start, end, nil, nil)
+}
+
+// NewDateRangeInclusiveQuery creates a new Query for ranges
+// of date values.
+// Date strings are parsed using the DateTimeParser configured for the
+// queried field in the index mapping.
+// Either, but not both endpoints can be nil.
+// startInclusive and endInclusive control inclusion of the endpoints.
+func NewDateRangeRawInclusiveQuery(start, end string, startInclusive, endInclusive *bool) *DateRangeQuery {
+	return &DateRangeQuery{
+		RawStart:       start,
+		RawEnd:         end,
+		InclusiveStart: startInclusive,
+		InclusiveEnd:   endInclusive,
+		InheritParser:  true,
+	}
+}
+
 func (q *DateRangeQuery) SetBoost(b float64) {
 	boost := Boost(b)
 	q.BoostVal = &boost
@@ -134,23 +236,48 @@ func (q *DateRangeQuery) Field() string {
 	return q.FieldVal
 }
 
-func (q *DateRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {
-	min, max, err := q.parseEndpoints()
-	if err != nil {
-		return nil, err
-	}
+func (q *DateRangeQuery) SetInheritParser(i bool) {
+	q.InheritParser = i
+}
 
+func (q *DateRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {
 	field := q.FieldVal
 	if q.FieldVal == "" {
 		field = m.DefaultSearchField()
 	}
-
+	if q.InheritParser {
+		var err error
+		// inherit parser from index mapping
+		dateTimeParserName := m.DatetimeParserNameForPath(field)
+		dateTimeParser := m.DateTimeParserNamed(dateTimeParserName)
+		if q.RawStart != "" {
+			q.Start.Time, _, err = dateTimeParser.ParseDateTime(q.RawStart)
+			if err != nil {
+				return nil, fmt.Errorf("%v, date time parser name: %s", err, dateTimeParserName)
+			}
+		}
+		if q.RawEnd != "" {
+			q.End.Time, _, err = dateTimeParser.ParseDateTime(q.RawEnd)
+			if err != nil {
+				return nil, fmt.Errorf("%v, date time parser name: %s", err, dateTimeParserName)
+			}
+		}
+	}
+	min, max, err := q.parseEndpoints()
+	if err != nil {
+		return nil, err
+	}
 	return searcher.NewNumericRangeSearcher(ctx, i, min, max, q.InclusiveStart, q.InclusiveEnd, field, q.BoostVal.Value(), options)
 }
 
 func (q *DateRangeQuery) parseEndpoints() (*float64, *float64, error) {
 	min := math.Inf(-1)
 	max := math.Inf(1)
+
+	if q.Start.IsZero() && q.End.IsZero() {
+		return nil, nil, fmt.Errorf("date range query must specify at least one of start/end")
+	}
+
 	if !q.Start.IsZero() {
 		if !isDatetimeCompatible(q.Start) {
 			// overflow
@@ -172,8 +299,17 @@ func (q *DateRangeQuery) parseEndpoints() (*float64, *float64, error) {
 }
 
 func (q *DateRangeQuery) Validate() error {
+	// either start or end must be specified
 	if q.Start.IsZero() && q.End.IsZero() {
-		return fmt.Errorf("must specify start or end")
+		// if inherit parser is true, perform check if RawStart/RawEnd is specified
+		if q.InheritParser {
+			if q.RawStart == "" && q.RawEnd == "" {
+				// Really invalid now
+				return fmt.Errorf("date range query must specify at least one of start/end")
+			}
+		} else {
+			return fmt.Errorf("date range query must specify at least one of start/end")
+		}
 	}
 	_, _, err := q.parseEndpoints()
 	if err != nil {