Skip to content

Commit

Permalink
change java to iso
Browse files Browse the repository at this point in the history
  • Loading branch information
@CascadingRadium @abhinavdangeti
CascadingRadium authored and abhinavdangeti committed Sep 13, 2023
1 parent 659d869 commit 0b1144e
Show file tree
Hide file tree
Showing 6 changed files with 220 additions and 39 deletions.
10 changes: 5 additions & 5 deletions analysis/datetime/javatime/javatime.go → analysis/datetime/iso/iso.go
View file
Expand Up @@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package javatime
package iso

import (
"fmt"
Expand All @@ -23,11 +23,11 @@ import (
"github.com/blevesearch/bleve/v2/registry"
)

const Name = "javastyle"
const Name = "isostyle"

var textLiteralDelimiter byte = '\'' // single quote

// java style date strings are represented in
// ISO style date strings are represented in
// https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html
//
// Some format specifiers are not specified in go time package, such as:
Expand Down Expand Up @@ -121,7 +121,7 @@ func invalidFormatError(character byte, count int) error {
return fmt.Errorf("invalid format string, unknown format specifier: " + strings.Repeat(string(character), count))
}

func parseJavaString(layout string) (string, error) {
func parseISOString(layout string) (string, error) {
var dateTimeLayout strings.Builder

for idx := 0; idx < len(layout); {
Expand Down Expand Up @@ -228,7 +228,7 @@ func DateTimeParserConstructor(config map[string]interface{}, cache *registry.Ca
for _, layout := range layouts {
layoutStr, ok := layout.(string)
if ok {
layout, err := parseJavaString(layoutStr)
layout, err := parseISOString(layoutStr)
if err != nil {
return nil, err
}
Expand Down
21 changes: 11 additions & 10 deletions analysis/datetime/javatime/javatime_test.go → analysis/datetime/iso/iso_test.go
View file
Expand Up @@ -12,14 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package javatime
package iso

import (
"fmt"
"testing"
)

func TestConversionFromJavaStyle(t *testing.T) {
func TestConversionFromISOStyle(t *testing.T) {
tests := []struct {
input string
output string
Expand All @@ -43,6 +43,7 @@ func TestConversionFromJavaStyle(t *testing.T) {
{
input: "MMMM dd yyyy', 'HH:mm:ss.SSS",
output: "January 02 2006, 15:04:05.000",
err: nil,
},
{
input: "h 'o'''' clock' a, XXX",
Expand All @@ -55,13 +56,8 @@ func TestConversionFromJavaStyle(t *testing.T) {
err: nil,
},
{
input: "E MMM d H:m:s z Y",
output: "Mon Jan 2 15:4:5 MST 2006",
err: nil,
},
{
input: "E MMM d H:m:s z Y",
output: "Mon Jan 2 15:4:5 MST 2006",
input: "E MMM d H:mm:ss z Y",
output: "Mon Jan 2 15:04:05 MST 2006",
err: nil,
},
{
Expand All @@ -74,9 +70,14 @@ func TestConversionFromJavaStyle(t *testing.T) {
output: "",
err: fmt.Errorf("invalid format string, expected text literal delimiter: '"),
},
{
input: "MMMMM dd yyyy', 'HH:mm:ss.SSS",
output: "",
err: fmt.Errorf("invalid format string, unknown format specifier: MMMMM"),
},
}
for _, test := range tests {
out, err := parseJavaString(test.input)
out, err := parseISOString(test.input)
if err != nil && test.err == nil || err == nil && test.err != nil {
t.Fatalf("expected error %v, got error %v", test.err, err)
}
Expand Down
43 changes: 21 additions & 22 deletions analysis/datetime/percent/percent.go
View file
Expand Up @@ -27,29 +27,28 @@ const Name = "percentstyle"

var formatDelimiter byte = '%'

// format specifiers as per strftime in the C standard library
// https://man7.org/linux/man-pages/man3/strftime.3.html
var formatSpecifierToLayout = map[byte]string{
// format specifiers as per strftime in the C standard library
// https://man7.org/linux/man-pages/man3/strftime.3.html

formatDelimiter: string(formatDelimiter),
'a': "Mon", // %a = short weekday name
'A': "Monday", // %A = full weekday name
'd': "02", // %d = day of month (2 digits) (01-31)
'e': "2", // %e = day of month (1 digit) (1-31)
'b': "Jan", // %b = short month name
'B': "January", // %B = full month name
'm': "01", // %m = month of year (2 digits) (01-12)
'y': "06", // %y = year without century
'Y': "2006", // %Y = year with century
'H': "15", // %H = hour (24 hour clock) (2 digits)
'I': "03", // %I = hour (12 hour clock) (2 digits)
'l': "3", // %l = hour (12 hour clock) (1 digit)
'p': "PM", // %p = PM/AM
'P': "pm", // %P = pm/am (lowercase)
'M': "04", // %M = minute (2 digits)
'S': "05", // %S = seconds (2 digits)
'f': "999999", // .%f = fraction of seconds - up to microseconds (6 digits) - deci/milli/micro
'Z': "MST", // %Z = timezone name (GMT, JST, UTC etc)
formatDelimiter: string(formatDelimiter), // %% = % (literal %)
'a': "Mon", // %a = short weekday name
'A': "Monday", // %A = full weekday name
'd': "02", // %d = day of month (2 digits) (01-31)
'e': "2", // %e = day of month (1 digit) (1-31)
'b': "Jan", // %b = short month name
'B': "January", // %B = full month name
'm': "01", // %m = month of year (2 digits) (01-12)
'y': "06", // %y = year without century
'Y': "2006", // %Y = year with century
'H': "15", // %H = hour (24 hour clock) (2 digits)
'I': "03", // %I = hour (12 hour clock) (2 digits)
'l': "3", // %l = hour (12 hour clock) (1 digit)
'p': "PM", // %p = PM/AM
'P': "pm", // %P = pm/am (lowercase)
'M': "04", // %M = minute (2 digits)
'S': "05", // %S = seconds (2 digits)
'f': "999999", // .%f = fraction of seconds - up to microseconds (6 digits) - deci/milli/micro
'Z': "MST", // %Z = timezone name (GMT, JST, UTC etc)
// %z is present in timezone options

// some additional options not in strftime to support additional options such as
Expand Down
2 changes: 1 addition & 1 deletion config/config.go
View file
Expand Up @@ -70,7 +70,7 @@ import (

// date time parsers
_ "github.com/blevesearch/bleve/v2/analysis/datetime/flexible"
_ "github.com/blevesearch/bleve/v2/analysis/datetime/javatime"
_ "github.com/blevesearch/bleve/v2/analysis/datetime/iso"
_ "github.com/blevesearch/bleve/v2/analysis/datetime/optional"
_ "github.com/blevesearch/bleve/v2/analysis/datetime/percent"
_ "github.com/blevesearch/bleve/v2/analysis/datetime/sanitized"
Expand Down
34 changes: 34 additions & 0 deletions query.go
View file
Expand Up @@ -68,6 +68,40 @@ func NewDateRangeInclusiveQuery(start, end time.Time, startInclusive, endInclusi
return query.NewDateRangeInclusiveQuery(start, end, startInclusive, endInclusive)
}

// NewDateRangeStringQuery creates a new Query for ranges
// of date values.
// Date strings are parsed using the DateTimeParser set using
//
// the DateRangeStringQuery.SetDateTimeParser() method.
//
// If no DateTimeParser is set, then the
//
// top-level config.QueryDateTimeParser
//
// is used.
func NewDateRangeStringQuery(start, end string) *query.DateRangeStringQuery {
return query.NewDateRangeStringQuery(start, end)
}

// NewDateRangeStringQuery creates a new Query for ranges
// of date values.
// Date strings are parsed using the DateTimeParser set using
//
// the DateRangeStringQuery.SetDateTimeParser() method.
//
// this DateTimeParser is a custom date time parser defined in the index mapping,
// using AddCustomDateTimeParser() method.
// If no DateTimeParser is set, then the
//
// top-level config.QueryDateTimeParser
//
// is used.
// Either, but not both endpoints can be nil.
// startInclusive and endInclusive control inclusion of the endpoints.
func NewDateRangeInclusiveStringQuery(start, end string, startInclusive, endInclusive *bool) *query.DateRangeStringQuery {
return query.NewDateRangeStringInclusiveQuery(start, end, startInclusive, endInclusive)
}

// NewDisjunctionQuery creates a new compound Query.
// Result documents satisfy at least one Query.
func NewDisjunctionQuery(disjuncts ...query.Query) *query.DisjunctionQuery {
Expand Down
149 changes: 148 additions & 1 deletion search_test.go
View file
Expand Up @@ -30,6 +30,8 @@ import (
html_char_filter "github.com/blevesearch/bleve/v2/analysis/char/html"
regexp_char_filter "github.com/blevesearch/bleve/v2/analysis/char/regexp"
"github.com/blevesearch/bleve/v2/analysis/datetime/flexible"
"github.com/blevesearch/bleve/v2/analysis/datetime/iso"
"github.com/blevesearch/bleve/v2/analysis/datetime/percent"
"github.com/blevesearch/bleve/v2/analysis/datetime/sanitized"
"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/microseconds"
"github.com/blevesearch/bleve/v2/analysis/datetime/timestamp/milliseconds"
Expand Down Expand Up @@ -2740,7 +2742,7 @@ func TestDateRangeStringQuery(t *testing.T) {

for _, dtq := range testQueries {
var err error
dateQuery := query.NewDateRangeStringInclusiveQuery(dtq.start, dtq.end, &dtq.includeStart, &dtq.includeEnd)
dateQuery := NewDateRangeInclusiveStringQuery(dtq.start, dtq.end, &dtq.includeStart, &dtq.includeEnd)
dateQuery.SetDateTimeParser(dtq.dateTimeParser)
dateQuery.SetField(dtq.field)

Expand Down Expand Up @@ -3229,3 +3231,148 @@ func TestDateRangeTimestampQueries(t *testing.T) {
}
}
}

func TestPercentAndIsoStyleDates(t *testing.T) {
percentName := percent.Name
isoName := iso.Name

imap := mapping.NewIndexMapping()
percentConfig := map[string]interface{}{
"type": percentName,
"layouts": []interface{}{
"%Y/%m/%d %l:%M%p", // doc 1
"%d/%m/%Y %H:%M:%S", // doc 2
"%Y-%m-%dT%H:%M:%S%z", // doc 3
"%d %B %y %l%p %Z", // doc 4
"%Y; %b %d (%a) %I:%M:%S.%N%P %z", // doc 5
},
}
isoConfig := map[string]interface{}{
"type": isoName,
"layouts": []interface{}{
"yyyy/MM/dd h:mma", // doc 1
"dd/MM/yyyy HH:mm:ss", // doc 2
"yyyy-MM-dd'T'HH:mm:ssXX", // doc 3
"dd MMMM yy ha z", // doc 4
"yyyy; MMM dd (EEE) hh:mm:ss.SSSSSaa xx", // doc 5
},
}

err := imap.AddCustomDateTimeParser("percentDate", percentConfig)
if err != nil {
t.Fatal(err)
}
err = imap.AddCustomDateTimeParser("isoDate", isoConfig)
if err != nil {
t.Fatal(err)
}

percentField := mapping.NewDateTimeFieldMapping()
percentField.DateFormat = "percentDate"

isoField := mapping.NewDateTimeFieldMapping()
isoField.DateFormat = "isoDate"

imap.DefaultMapping.AddFieldMappingsAt("percentDate", percentField)
imap.DefaultMapping.AddFieldMappingsAt("isoDate", isoField)

tmpIndexPath := createTmpIndexPath(t)
defer cleanupTmpIndexPath(t, tmpIndexPath)

idx, err := New(tmpIndexPath, imap)
if err != nil {
t.Fatal(err)
}
defer func() {
err = idx.Close()
if err != nil {
t.Fatal(err)
}
}()

documents := map[string]map[string]interface{}{
"doc1": {
"percentDate": "2001/08/20 6:00PM",
"isoDate": "2001/08/20 6:00PM",
},
"doc2": {
"percentDate": "20/08/2001 18:05:00",
"isoDate": "20/08/2001 18:05:00",
},
"doc3": {
"percentDate": "2001-08-20T18:10:00Z",
"isoDate": "2001-08-20T18:10:00Z",
},
"doc4": {
"percentDate": "20 August 01 6PM UTC",
"isoDate": "20 August 01 6PM UTC",
},
"doc5": {
"percentDate": "2001; Aug 20 (Mon) 06:15:15.23456pm +0000",
"isoDate": "2001; Aug 20 (Mon) 06:15:15.23456pm +0000",
},
}

batch := idx.NewBatch()
for docID, doc := range documents {
err := batch.Index(docID, doc)
if err != nil {
t.Fatal(err)
}
}
err = idx.Batch(batch)
if err != nil {
t.Fatal(err)
}

type testStruct struct {
start string
end string
field string
}

for _, field := range []string{"percentDate", "isoDate"} {
testQueries := []testStruct{
{
start: "2001/08/20 6:00PM",
end: "2001/08/20 6:20PM",
field: field,
},
{
start: "20/08/2001 18:00:00",
end: "20/08/2001 18:20:00",
field: field,
},
{
start: "2001-08-20T18:00:00Z",
end: "2001-08-20T18:20:00Z",
field: field,
},
{
start: "20 August 01 6PM UTC",
end: "20 August 01 7PM UTC",
field: field,
},
{
start: "2001; Aug 20 (Mon) 06:00:00.00000pm +0000",
end: "2001; Aug 20 (Mon) 06:20:20.00000pm +0000",
field: field,
},
}
includeStart := true
includeEnd := true
for _, dtq := range testQueries {
drq := NewDateRangeInclusiveStringQuery(dtq.start, dtq.end, &includeStart, &includeEnd)
drq.SetField(dtq.field)
drq.SetDateTimeParser(field)
sr := NewSearchRequest(drq)
res, err := idx.Search(sr)
if err != nil {
t.Fatal(err)
}
if len(res.Hits) != 5 {
t.Fatalf("expected %d hits, got %d", 5, len(res.Hits))
}
}
}
}

0 comments on commit 0b1144e

Please sign in to comment.