Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

MB-58388: Use custom datetime parsers in date range facet queries #1878

Merged
merged 5 commits into from Sep 7, 2023
Merged

MB-58388: Use custom datetime parsers in date range facet queries #1878

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3d3ee2c
retain layout
CascadingRadium Sep 5, 2023
833f787
query override
CascadingRadium Sep 5, 2023
988b133
unit test cases
CascadingRadium Sep 6, 2023
0f34de6
Merge branch 'master' into MB-58388
CascadingRadium Sep 7, 2023
1204111
fix bug
CascadingRadium Sep 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
19 changes: 16 additions & 3 deletions index_impl.go
View file
Open in desktop
Expand Up @@ -530,10 +530,23 @@ func (i *indexImpl) SearchInContext(ctx context.Context, req *SearchRequest) (sr
} else if facetRequest.DateTimeRanges != nil {
// build date range facet
facetBuilder := facet.NewDateTimeFacetBuilder(facetRequest.Field, facetRequest.Size)
dateTimeParser := i.m.DateTimeParserNamed("")
for _, dr := range facetRequest.DateTimeRanges {
start, end := dr.ParseDates(dateTimeParser)
facetBuilder.AddRange(dr.Name, start, end)
dateTimeParserName := defaultDateTimeParser
if dr.DateTimeParser != "" {
dateTimeParserName = dr.DateTimeParser
CascadingRadium marked this conversation as resolved.
Show resolved Hide resolved
}
dateTimeParser := i.m.DateTimeParserNamed(dateTimeParserName)
if dateTimeParser == nil {
return nil, fmt.Errorf("no date time parser named `%s` registered", dateTimeParserName)
abhinavdangeti marked this conversation as resolved.
Show resolved Hide resolved
}
start, end, startLayout, endLayout, err := dr.ParseDates(dateTimeParser)
if err != nil {
return nil, fmt.Errorf("ParseDates err: %v, using date time parser named %s", err, dateTimeParserName)
}
if start.IsZero() && end.IsZero() {
return nil, fmt.Errorf("date range query must specify either start, end or both for date range name '%s'", dr.Name)
}
facetBuilder.AddRange(dr.Name, start, end, startLayout, endLayout)
}
facetsBuilder.Add(facetName, facetBuilder)
} else {
Expand Down
89 changes: 63 additions & 26 deletions search.go
View file
Open in desktop
Expand Up @@ -46,36 +46,44 @@ func init() {
}

type dateTimeRange struct {
Name string `json:"name,omitempty"`
Start time.Time `json:"start,omitempty"`
End time.Time `json:"end,omitempty"`
startString *string
endString *string
Name string `json:"name,omitempty"`
Start time.Time `json:"start,omitempty"`
End time.Time `json:"end,omitempty"`
DateTimeParser string `json:"datetime_parser,omitempty"`
startString *string
endString *string
}

func (dr *dateTimeRange) ParseDates(dateTimeParser analysis.DateTimeParser) (start, end time.Time) {
func (dr *dateTimeRange) ParseDates(dateTimeParser analysis.DateTimeParser) (start, end time.Time, startLayout, endLayout string, err error) {
start = dr.Start
startLayout = time.RFC3339Nano
if dr.Start.IsZero() && dr.startString != nil {
s, _, err := dateTimeParser.ParseDateTime(*dr.startString)
if err == nil {
start = s
s, layout, parseError := dateTimeParser.ParseDateTime(*dr.startString)
if parseError != nil {
return start, end, startLayout, endLayout, fmt.Errorf("error parsing start date '%s' for date range name '%s': %v", *dr.startString, dr.Name, parseError)
abhinavdangeti marked this conversation as resolved.
Show resolved Hide resolved
}
start = s
startLayout = layout
}
end = dr.End
endLayout = time.RFC3339Nano
if dr.End.IsZero() && dr.endString != nil {
e, _, err := dateTimeParser.ParseDateTime(*dr.endString)
if err == nil {
end = e
e, layout, parseError := dateTimeParser.ParseDateTime(*dr.endString)
if parseError != nil {
return start, end, startLayout, endLayout, fmt.Errorf("error parsing end date '%s' for date range name '%s': %v", *dr.endString, dr.Name, parseError)
abhinavdangeti marked this conversation as resolved.
Show resolved Hide resolved
}
end = e
endLayout = layout
}
return start, end
return start, end, startLayout, endLayout, err
}

func (dr *dateTimeRange) UnmarshalJSON(input []byte) error {
var temp struct {
Name string `json:"name,omitempty"`
Start *string `json:"start,omitempty"`
End *string `json:"end,omitempty"`
Name string `json:"name,omitempty"`
Start *string `json:"start,omitempty"`
End *string `json:"end,omitempty"`
DateTimeParser string `json:"datetime_parser,omitempty"`
}

if err := json.Unmarshal(input, &temp); err != nil {
Expand All @@ -89,22 +97,33 @@ func (dr *dateTimeRange) UnmarshalJSON(input []byte) error {
if temp.End != nil {
dr.endString = temp.End
}
if temp.DateTimeParser != "" {
dr.DateTimeParser = temp.DateTimeParser
}

return nil
}

func (dr *dateTimeRange) MarshalJSON() ([]byte, error) {
rv := map[string]interface{}{
"name": dr.Name,
"start": dr.Start,
"end": dr.End,
"name": dr.Name,
}
if dr.Start.IsZero() && dr.startString != nil {

if !dr.Start.IsZero() {
rv["start"] = dr.Start
} else if dr.startString != nil {
rv["start"] = dr.startString
}
if dr.End.IsZero() && dr.endString != nil {

if !dr.End.IsZero() {
rv["end"] = dr.End
} else if dr.endString != nil {
rv["end"] = dr.endString
}

if dr.DateTimeParser != "" {
rv["datetime_parser"] = dr.DateTimeParser
}
return json.Marshal(rv)
}

Expand Down Expand Up @@ -138,7 +157,7 @@ func (fr *FacetRequest) Validate() error {
nrCount := len(fr.NumericRanges)
drCount := len(fr.DateTimeRanges)
if nrCount > 0 && drCount > 0 {
return fmt.Errorf("facet can only conain numeric ranges or date ranges, not both")
return fmt.Errorf("facet can only contain numeric ranges or date ranges, not both")
}

if nrCount > 0 {
Expand All @@ -164,9 +183,16 @@ func (fr *FacetRequest) Validate() error {
return fmt.Errorf("date ranges contains duplicate name '%s'", dr.Name)
}
drNames[dr.Name] = struct{}{}
start, end := dr.ParseDates(dateTimeParser)
if start.IsZero() && end.IsZero() {
return fmt.Errorf("date range query must specify either start, end or both for range name '%s'", dr.Name)
if dr.DateTimeParser == "" {
// cannot parse the date range dates as the defaultDateTimeParser is overridden
// so perform this validation at query time
start, end, _, _, err := dr.ParseDates(dateTimeParser)
if err != nil {
return fmt.Errorf("ParseDates err: %v, using date time parser named %s", err, defaultDateTimeParser)
}
if start.IsZero() && end.IsZero() {
return fmt.Errorf("date range query must specify either start, end or both for range name '%s'", dr.Name)
}
}
}
}
Expand All @@ -186,7 +212,7 @@ func (fr *FacetRequest) AddDateTimeRange(name string, start, end time.Time) {
}

// AddDateTimeRangeString adds a bucket to a field
// containing date values.
// containing date values. Uses defaultDateTimeParser to parse the date strings.
func (fr *FacetRequest) AddDateTimeRangeString(name string, start, end *string) {
if fr.DateTimeRanges == nil {
fr.DateTimeRanges = make([]*dateTimeRange, 0, 1)
Expand All @@ -195,6 +221,17 @@ func (fr *FacetRequest) AddDateTimeRangeString(name string, start, end *string)
&dateTimeRange{Name: name, startString: start, endString: end})
}

// AddDateTimeRangeString adds a bucket to a field
// containing date values. Uses the specified parser to parse the date strings.
// provided the parser is registered in the index mapping.
func (fr *FacetRequest) AddDateTimeRangeStringWithParser(name string, start, end *string, parser string) {
if fr.DateTimeRanges == nil {
fr.DateTimeRanges = make([]*dateTimeRange, 0, 1)
}
fr.DateTimeRanges = append(fr.DateTimeRanges,
&dateTimeRange{Name: name, startString: start, endString: end, DateTimeParser: parser})
}

// AddNumericRange adds a bucket to a field
// containing numeric values. Documents with a
// numeric value falling into this range are
Expand Down
31 changes: 24 additions & 7 deletions search/facet/facet_builder_datetime.go
View file
Open in desktop
Expand Up @@ -17,6 +17,7 @@ package facet
import (
"reflect"
"sort"
"strconv"
"time"

"github.com/blevesearch/bleve/v2/numeric"
Expand All @@ -35,8 +36,10 @@ func init() {
}

type dateTimeRange struct {
start time.Time
end time.Time
start time.Time
end time.Time
startLayout string
endLayout string
}

type DateTimeFacetBuilder struct {
Expand Down Expand Up @@ -75,10 +78,12 @@ func (fb *DateTimeFacetBuilder) Size() int {
return sizeInBytes
}

func (fb *DateTimeFacetBuilder) AddRange(name string, start, end time.Time) {
func (fb *DateTimeFacetBuilder) AddRange(name string, start, end time.Time, startLayout string, endLayout string) {
r := dateTimeRange{
start: start,
end: end,
start: start,
end: end,
startLayout: startLayout,
endLayout: endLayout,
}
fb.ranges[name] = &r
}
Expand Down Expand Up @@ -134,11 +139,23 @@ func (fb *DateTimeFacetBuilder) Result() *search.FacetResult {
Count: count,
}
if !dateRange.start.IsZero() {
start := dateRange.start.Format(time.RFC3339Nano)
var start string
if dateRange.startLayout == "" {
// layout not set probably means it is probably a timestamp
start = strconv.FormatInt(dateRange.start.UnixNano(), 10)
} else {
start = dateRange.start.Format(dateRange.startLayout)
}
tf.Start = &start
}
if !dateRange.end.IsZero() {
end := dateRange.end.Format(time.RFC3339Nano)
var end string
if dateRange.endLayout == "" {
// layout not set probably means it is probably a timestamp
end = strconv.FormatInt(dateRange.end.UnixNano(), 10)
} else {
end = dateRange.end.Format(dateRange.endLayout)
}
tf.End = &end
}
rv.DateRanges = append(rv.DateRanges, tf)
Expand Down