feat: Support for range queries on time fields (#429)

* refactor: Index datetime fields as int64

* fix: addressing PR feedback

* refactor: Remove schema dependency from lib

lib/date/converter.go

@@ -0,0 +1,14 @@
+package date
+
+import (
+	"time"
+)
+
+// ToUnixNano converts a time to Unix nano seconds
+func ToUnixNano(format string, dateStr string) (int64, error) {
+	t, err := time.Parse(format, dateStr)
+	if err != nil {
+		return 0, err
+	}
+	return t.UnixNano(), nil
+}

lib/date/converter_test.go

@@ -0,0 +1,45 @@
+package date
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"time"
+)
+
+func TestToUnixNano(t *testing.T) {
+	validCases := []struct {
+		name     string
+		date     string
+		expected int64
+	}{
+		{"UTC RFC 3339", "2022-10-18T00:51:07+00:00", 1666054267000000000},
+		{"UTC RFC 3339 Nano", "2022-10-18T00:51:07.528106+00:00", 1666054267528106000},
+		{"IST RFC 3339", "2022-10-11T04:19:32+05:30", 1665442172000000000},
+		{"IST RFC 3339 Nano", "2022-10-18T00:51:07.999999999+05:30", 1666034467999999999},
+		{"No TZ RFC 3339", "2022-10-18T00:51:07Z", 1666054267000000000},
+	}
+
+	for _, v := range validCases {
+		t.Run(v.name, func(t *testing.T) {
+			actual, err := ToUnixNano(time.RFC3339Nano, v.date)
+			assert.NoError(t, err)
+			assert.Equal(t, v.expected, actual)
+		})
+	}
+
+	failureCases := []struct {
+		name      string
+		date      string
+		errorLike string
+	}{
+		{"RFC 1123", "Mon, 02 Jan 2006 15:04:05 MST", "cannot parse"},
+	}
+
+	for _, v := range failureCases {
+		t.Run(v.name, func(t *testing.T) {
+			_, err := ToUnixNano(time.RFC3339Nano, v.date)
+			assert.ErrorContains(t, err, v.errorLike)
+		})
+	}
+}

query/filter/selector.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/tigrisdata/tigris/schema"
 	"github.com/tigrisdata/tigris/value"
+	"github.com/tigrisdata/tigris/lib/date"
 )
 
 // Selector is a condition defined inside a filter. It has a field which corresponding the field on which condition
@@ -88,6 +89,12 @@ func (s *Selector) ToSearchFilter() []string {
 	case schema.DoubleType:
 		// for double, we pass string in the filter to search backend
 		return []string{fmt.Sprintf(op, s.Field.Name(), v.String())}
+	case schema.DateTimeType:
+		// encode into int64
+		if nsec, err := date.ToUnixNano(schema.DateTimeFormat, v.String()); err == nil {
+			return []string{fmt.Sprintf(op, s.Field.Name(), nsec)}
+		}
+
 	}
 	return []string{fmt.Sprintf(op, s.Field.Name(), v.AsInterface())}
 }

schema/collection.go

@@ -184,16 +184,27 @@ func GetSearchDeltaFields(existingFields []*QueryableField, incomingFields []*Fi
 }
 
 func buildSearchSchema(name string, queryableFields []*QueryableField) *tsApi.CollectionSchema {
-	var ptrTrue = true
+	var ptrTrue, ptrFalse = true, false
 	var tsFields []tsApi.Field
 	for _, s := range queryableFields {
 		tsFields = append(tsFields, tsApi.Field{
-			Name:     s.FieldName,
+			Name:     s.Name(),
 			Type:     s.SearchType,
 			Facet:    &s.Faceted,
 			Index:    &s.Indexed,
 			Optional: &ptrTrue,
 		})
+		// Save original date as string to disk
+		if s.DataType == DateTimeType {
+			tsFields = append(tsFields, tsApi.Field{
+				Name:     ToSearchDateKey(s.Name()),
+				Type:     toSearchFieldType(StringType),
+				Facet:    &ptrFalse,
+				Index:    &ptrFalse,
+				Sort:     &ptrFalse,
+				Optional: &ptrTrue,
+			})
+		}
 	}
 
 	return &tsApi.CollectionSchema{

schema/collection_test.go

@@ -282,7 +282,7 @@ func TestCollection_SearchSchema(t *testing.T) {
 	schFactory, err := Build("t1", reqSchema)
 	require.NoError(t, err)
 
-	expFlattenedFields := []string{"id", "id_32", "product", "id_uuid", "ts", "price", "simple_items", "simple_object.name",
+	expFlattenedFields := []string{"id", "id_32", "product", "id_uuid", "ts", ToSearchDateKey("ts"), "price", "simple_items", "simple_object.name",
 		"simple_object.phone", "simple_object.address.street", "simple_object.details.nested_id", "simple_object.details.nested_obj.id",
 		"simple_object.details.nested_obj.name", "simple_object.details.nested_array", "simple_object.details.nested_string",
 	}

schema/fields.go

@@ -185,8 +185,10 @@ func toSearchFieldType(fieldType FieldType) string {
 		return FieldNames[fieldType]
 	case Int32Type, Int64Type:
 		return FieldNames[fieldType]
-	case StringType, ByteType, UUIDType, DateTimeType:
+	case StringType, ByteType, UUIDType:
 		return FieldNames[StringType]
+	case DateTimeType:
+		return FieldNames[Int64Type]
 	case DoubleType:
 		return searchDoubleType
 	case ArrayType:
@@ -393,7 +395,7 @@ func (q *QueryableField) Name() string {
 }
 
 func (q *QueryableField) ShouldPack() bool {
-	return q.DataType == ArrayType
+	return q.DataType == ArrayType || q.DataType == DateTimeType
 }
 
 func buildQueryableFields(fields []*Field) []*QueryableField {

schema/reserved.go

@@ -21,13 +21,15 @@ const (
 	UpdatedAt
 	Metadata
 	IdToSearchKey
+	DateSearchKeyPrefix
 )
 
 var ReservedFields = [...]string{
-	CreatedAt:     "created_at",
-	UpdatedAt:     "updated_at",
-	Metadata:      "metadata",
-	IdToSearchKey: "_tigris_id",
+	CreatedAt:           "created_at",
+	UpdatedAt:           "updated_at",
+	Metadata:            "metadata",
+	IdToSearchKey:       "_tigris_id",
+	DateSearchKeyPrefix: "_tigris_date_",
 }
 
 func IsReservedField(name string) bool {
@@ -39,3 +41,9 @@ func IsReservedField(name string) bool {
 
 	return false
 }
+
+// ToSearchDateKey can be used to generate storage field for search backend
+// Original date strings are persisted as it is under this field
+func ToSearchDateKey(key string) string {
+	return ReservedFields[DateSearchKeyPrefix] + key
+}

schema/schema.go

@@ -15,6 +15,8 @@
 package schema
 
 import (
+	"time"
+
 	"github.com/buger/jsonparser"
 	jsoniter "github.com/json-iterator/go"
 	"github.com/pkg/errors"
@@ -66,6 +68,8 @@ const (
 	PrimaryKeyIndexName = "pkey"
 	AutoPrimaryKeyF     = "id"
 	PrimaryKeySchemaK   = "primary_key"
+	// DateTimeFormat represents the supported date time format
+	DateTimeFormat = time.RFC3339Nano
 )
 
 var (

server/services/v1/search_indexer.go

@@ -24,7 +24,9 @@ import (
 	"github.com/apple/foundationdb/bindings/go/src/fdb"
 	"github.com/apple/foundationdb/bindings/go/src/fdb/subspace"
 	jsoniter "github.com/json-iterator/go"
+	api "github.com/tigrisdata/tigris/api/server/v1"
 	"github.com/tigrisdata/tigris/internal"
+	"github.com/tigrisdata/tigris/lib/date"
 	"github.com/tigrisdata/tigris/lib/json"
 	"github.com/tigrisdata/tigris/schema"
 	"github.com/tigrisdata/tigris/server/metadata"
@@ -179,12 +181,30 @@ func PackSearchFields(data *internal.TableData, collection *schema.DefaultCollec
 
 	decData = FlattenObjects(decData)
 
-	// now if there is any array we need to pack it
-	for key, value := range decData {
-		if _, ok := value.([]any); ok {
-			// pack any array field
-			if decData[key], err = jsoniter.MarshalToString(value); err != nil {
-				return nil, err
+	// pack any date time or array fields here
+	for _, f := range collection.QueryableFields {
+		key, value := f.Name(), decData[f.Name()]
+		if value == nil {
+			continue
+		}
+		if f.ShouldPack() {
+			switch f.DataType {
+			case schema.ArrayType:
+				if decData[key], err = jsoniter.MarshalToString(value); err != nil {
+					return nil, err
+				}
+			case schema.DateTimeType:
+				if dateStr, ok := value.(string); ok {
+					t, err := date.ToUnixNano(schema.DateTimeFormat, dateStr)
+					if err != nil {
+						return nil, api.Errorf(api.Code_INVALID_ARGUMENT, "Validation failed, %s is not a valid date-time", dateStr)
+					}
+					decData[key] = t
+					// pack original date as string to a shadowed key
+					decData[schema.ToSearchDateKey(key)] = dateStr
+				}
+			default:
+				return nil, api.Errorf(api.Code_UNIMPLEMENTED, "Internal error!")
 			}
 		}
 	}
@@ -207,11 +227,21 @@ func UnpackSearchFields(doc map[string]interface{}, collection *schema.DefaultCo
 	for _, f := range collection.QueryableFields {
 		if f.ShouldPack() {
 			if v, ok := doc[f.Name()]; ok {
-				var value interface{}
-				if err := jsoniter.UnmarshalFromString(v.(string), &value); err != nil {
-					return "", nil, nil, err
+				switch f.DataType {
+				case schema.ArrayType:
+					var value interface{}
+					if err := jsoniter.UnmarshalFromString(v.(string), &value); err != nil {
+						return "", nil, nil, err
+					}
+					doc[f.Name()] = value
+				case schema.DateTimeType:
+					// unpack original date from shadowed key
+					shadowedKey := schema.ToSearchDateKey(f.Name())
+					doc[f.Name()] = doc[shadowedKey]
+					delete(doc, shadowedKey)
+				default:
+					return "", nil, nil, api.Errorf(api.Code_UNIMPLEMENTED, "Internal error!")
 				}
-				doc[f.Name()] = value
 			}
 		}
 	}