From 299aa0515f862e7319fa6acd175238d91cf414d5 Mon Sep 17 00:00:00 2001 From: Shinu Joseph Date: Sun, 7 Dec 2025 16:32:33 +0100 Subject: [PATCH 1/5] feat: extend filtering for Qdrant searches --- v1/qdrant/filters.go | 238 +++++++++++++++++ v1/qdrant/filters_test.go | 531 ++++++++++++++++++++++++++++++++++++++ v1/qdrant/utils.go | 50 +--- 3 files changed, 780 insertions(+), 39 deletions(-) create mode 100644 v1/qdrant/filters.go create mode 100644 v1/qdrant/filters_test.go diff --git a/v1/qdrant/filters.go b/v1/qdrant/filters.go new file mode 100644 index 0000000..be04866 --- /dev/null +++ b/v1/qdrant/filters.go @@ -0,0 +1,238 @@ +package qdrant + +import ( + "strings" + "time" + + qdrant "github.com/qdrant/go-client/qdrant" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// UserPayloadPrefix is the prefix for user-defined metadata fields +const UserPayloadPrefix = "custom" + +// FilterCondition is the interface for all filter conditions +type FilterCondition interface { + toQdrantCondition() []*qdrant.Condition +} + +// FieldType indicates whether a field is internal or user-defined +type FieldType int + +const ( + // InternalField - system-managed fields stored at top-level + InternalField FieldType = iota + // UserField - user-defined fields stored under "custom." prefix + UserField +) + +// TimeRange represents a time-based filter condition +type TimeRange struct { + Gt *time.Time // Greater than this time + Gte *time.Time // Greater than or equal to this time + Lt *time.Time // Less than this time + Lte *time.Time // Less than or equal to this time +} + +type MatchCondition[T comparable] struct { + Key string + Value T + FieldType FieldType // Internal or User field (default: InternalField) +} + +func (c MatchCondition[T]) toQdrantCondition() []*qdrant.Condition { + key := resolveFieldKey(c.Key, c.FieldType) + switch v := any(c.Value).(type) { + case string: + return []*qdrant.Condition{qdrant.NewMatch(key, v)} + case bool: + return []*qdrant.Condition{qdrant.NewMatchBool(key, v)} + case int64: + return []*qdrant.Condition{qdrant.NewMatchInt(key, v)} + default: + //Unsupported type + return nil + } + +} + +// MatchAnyCondition matches if value is one of the given values (IN operator) +// Applicable to keyword (string) and integer payloads +type MatchAnyCondition[T string | int64] struct { + Key string + Values []T + FieldType FieldType +} + +func (c MatchAnyCondition[T]) toQdrantCondition() []*qdrant.Condition { + key := resolveFieldKey(c.Key, c.FieldType) + switch v := any(c.Values).(type) { + case []string: + return []*qdrant.Condition{qdrant.NewMatchKeywords(key, v...)} + case []int64: + return []*qdrant.Condition{qdrant.NewMatchInts(key, v...)} + default: + return nil + } +} + +// MatchExceptCondition matches if value is NOT one of the given values (NOT IN operator) +// Applicable to keyword (string) and integer payloads +type MatchExceptCondition[T string | int64] struct { + Key string + Values []T + FieldType FieldType +} + +func (c MatchExceptCondition[T]) toQdrantCondition() []*qdrant.Condition { + key := resolveFieldKey(c.Key, c.FieldType) + switch v := any(c.Values).(type) { + case []string: + return []*qdrant.Condition{qdrant.NewMatchExceptKeywords(key, v...)} + case []int64: + return []*qdrant.Condition{qdrant.NewMatchExceptInts(key, v...)} + default: + return nil + } +} + +type TextCondition = MatchCondition[string] +type BoolCondition = MatchCondition[bool] +type IntCondition = MatchCondition[int64] +type TextAnyCondition = MatchAnyCondition[string] +type IntAnyCondition = MatchAnyCondition[int64] +type TextExceptCondition = MatchExceptCondition[string] +type IntExceptCondition = MatchExceptCondition[int64] + +// TimeRangeCondition represents a time range filter condition +type TimeRangeCondition struct { + Key string + Value TimeRange + FieldType FieldType // Internal or User field (default: InternalField) +} + +func (c TimeRangeCondition) toQdrantCondition() []*qdrant.Condition { + return buildDateTimeRangeConditions(resolveFieldKey(c.Key, c.FieldType), c.Value) +} + +// resolveFieldKey returns the full field path based on FieldType +// Internal fields: "search_store_id" -> "search_store_id" +// User fields: "document_id" -> "custom.document_id" +func resolveFieldKey(key string, fieldType FieldType) string { + if fieldType == UserField { + // Prevent double-prefixing + if strings.HasPrefix(key, UserPayloadPrefix+".") { + return key + } + return UserPayloadPrefix + "." + key + } + return key +} + +// ConditionSet holds conditions for a single clause +type ConditionSet struct { + Conditions []FilterCondition +} + +// FilterSet supports Must (AND), Should (OR), and MustNot (NOT) clauses. +// Use with SearchRequest.Filters to filter search results. +// +// Example: +// +// filters := &FilterSet{ +// Must: &ConditionSet{ +// Conditions: []FilterCondition{ +// TextCondition{Key: "city", Value: "London"}, +// }, +// }, +// } +type FilterSet struct { + Must *ConditionSet // AND - all conditions must match + Should *ConditionSet // OR - at least one condition must match + MustNot *ConditionSet // NOT - none of the conditions should match +} + +// buildFilter constructs a Qdrant filter from FilterSet +func buildFilter(filters *FilterSet) *qdrant.Filter { + if filters == nil { + return nil + } + + filter := &qdrant.Filter{} + + if filters.Must != nil { + filter.Must = buildConditions(filters.Must) + } + + if filters.Should != nil { + filter.Should = buildConditions(filters.Should) + } + + if filters.MustNot != nil { + filter.MustNot = buildConditions(filters.MustNot) + } + + // Return nil if no conditions were added + if len(filter.Must) == 0 && len(filter.Should) == 0 && len(filter.MustNot) == 0 { + return nil + } + + return filter +} + +// buildConditions converts a ConditionSet to Qdrant conditions +func buildConditions(cs *ConditionSet) []*qdrant.Condition { + if cs == nil { + return nil + } + + var conditions []*qdrant.Condition + for _, c := range cs.Conditions { + conditions = append(conditions, c.toQdrantCondition()...) + } + return conditions +} + +// buildDateTimeRangeConditions creates datetime range conditions +func buildDateTimeRangeConditions(key string, tr TimeRange) []*qdrant.Condition { + dateRange := &qdrant.DatetimeRange{ + Gt: toTimestamp(tr.Gt), + Gte: toTimestamp(tr.Gte), + Lt: toTimestamp(tr.Lt), + Lte: toTimestamp(tr.Lte), + } + + // Check if any field is set + if dateRange.Gt == nil && dateRange.Gte == nil && dateRange.Lt == nil && dateRange.Lte == nil { + return nil + } + + return []*qdrant.Condition{qdrant.NewDatetimeRange(key, dateRange)} +} + +// toTimestamp converts a *time.Time to *timestamppb.Timestamp (nil-safe) +func toTimestamp(t *time.Time) *timestamppb.Timestamp { + if t == nil { + return nil + } + return timestamppb.New(*t) +} + +// === Payload Helpers === + +// BuildPayload creates a Qdrant payload with separated internal and user fields +func BuildPayload(internal map[string]any, user map[string]any) map[string]any { + payload := make(map[string]any) + + // Add internal fields at top-level + for k, v := range internal { + payload[k] = v + } + + // Add user fields under "custom" prefix + if len(user) > 0 { + payload[UserPayloadPrefix] = user + } + + return payload +} diff --git a/v1/qdrant/filters_test.go b/v1/qdrant/filters_test.go new file mode 100644 index 0000000..3812968 --- /dev/null +++ b/v1/qdrant/filters_test.go @@ -0,0 +1,531 @@ +package qdrant + +import ( + "testing" + "time" +) + +func TestBuildFilter_NilFilterSet(t *testing.T) { + result := buildFilter(nil) + if result != nil { + t.Errorf("expected nil, got %v", result) + } +} + +func TestBuildFilter_EmptyFilterSet(t *testing.T) { + filters := &FilterSet{} + result := buildFilter(filters) + if result != nil { + t.Errorf("expected nil, got %v", result) + } +} + +func TestBuildFilter_EmptyConditionSet(t *testing.T) { + filters := &FilterSet{ + Must: &ConditionSet{ + Conditions: []FilterCondition{}, + }, + } + result := buildFilter(filters) + if result != nil { + t.Errorf("expected nil, got %v", result) + } +} + +func TestBuildFilter_MustWithTextCondition(t *testing.T) { + filters := &FilterSet{ + Must: &ConditionSet{ + Conditions: []FilterCondition{ + TextCondition{Key: "city", Value: "London"}, + }, + }, + } + result := buildFilter(filters) + + if result == nil { + t.Fatal("expected filter, got nil") + } + if len(result.Must) != 1 { + t.Errorf("expected 1 Must condition, got %d", len(result.Must)) + } + if len(result.Should) != 0 { + t.Errorf("expected 0 Should conditions, got %d", len(result.Should)) + } + if len(result.MustNot) != 0 { + t.Errorf("expected 0 MustNot conditions, got %d", len(result.MustNot)) + } +} + +func TestBuildFilter_ShouldWithMultipleTextConditions(t *testing.T) { + // city = "London" OR city = "Berlin" + filters := &FilterSet{ + Should: &ConditionSet{ + Conditions: []FilterCondition{ + TextCondition{Key: "city", Value: "London"}, + TextCondition{Key: "city", Value: "Berlin"}, + }, + }, + } + result := buildFilter(filters) + + if result == nil { + t.Fatal("expected filter, got nil") + } + if len(result.Should) != 2 { + t.Errorf("expected 2 Should conditions, got %d", len(result.Should)) + } +} + +func TestBuildFilter_MustNotWithBoolCondition(t *testing.T) { + filters := &FilterSet{ + MustNot: &ConditionSet{ + Conditions: []FilterCondition{ + BoolCondition{Key: "archived", Value: true}, + }, + }, + } + result := buildFilter(filters) + + if result == nil { + t.Fatal("expected filter, got nil") + } + if len(result.MustNot) != 1 { + t.Errorf("expected 1 MustNot condition, got %d", len(result.MustNot)) + } +} + +func TestBuildFilter_MixedConditionTypes(t *testing.T) { + // city = "London" AND active = true AND priority = 1 + filters := &FilterSet{ + Must: &ConditionSet{ + Conditions: []FilterCondition{ + TextCondition{Key: "city", Value: "London"}, + BoolCondition{Key: "active", Value: true}, + IntCondition{Key: "priority", Value: 1}, + }, + }, + } + result := buildFilter(filters) + + if result == nil { + t.Fatal("expected filter, got nil") + } + if len(result.Must) != 3 { + t.Errorf("expected 3 Must conditions, got %d", len(result.Must)) + } +} + +func TestBuildFilter_CombinedClauses(t *testing.T) { + // (city = "London" AND active = true) AND NOT archived + filters := &FilterSet{ + Must: &ConditionSet{ + Conditions: []FilterCondition{ + TextCondition{Key: "city", Value: "London"}, + BoolCondition{Key: "active", Value: true}, + }, + }, + MustNot: &ConditionSet{ + Conditions: []FilterCondition{ + BoolCondition{Key: "archived", Value: true}, + }, + }, + } + result := buildFilter(filters) + + if result == nil { + t.Fatal("expected filter, got nil") + } + if len(result.Must) != 2 { + t.Errorf("expected 2 Must conditions, got %d", len(result.Must)) + } + if len(result.MustNot) != 1 { + t.Errorf("expected 1 MustNot condition, got %d", len(result.MustNot)) + } +} + +func TestBuildFilter_AllThreeClauses(t *testing.T) { + // Must AND Should AND MustNot + filters := &FilterSet{ + Must: &ConditionSet{ + Conditions: []FilterCondition{ + TextCondition{Key: "status", Value: "active"}, + }, + }, + Should: &ConditionSet{ + Conditions: []FilterCondition{ + TextCondition{Key: "city", Value: "London"}, + TextCondition{Key: "city", Value: "Berlin"}, + }, + }, + MustNot: &ConditionSet{ + Conditions: []FilterCondition{ + BoolCondition{Key: "deleted", Value: true}, + }, + }, + } + result := buildFilter(filters) + + if result == nil { + t.Fatal("expected filter, got nil") + } + if len(result.Must) != 1 { + t.Errorf("expected 1 Must condition, got %d", len(result.Must)) + } + if len(result.Should) != 2 { + t.Errorf("expected 2 Should conditions, got %d", len(result.Should)) + } + if len(result.MustNot) != 1 { + t.Errorf("expected 1 MustNot condition, got %d", len(result.MustNot)) + } +} + +func TestBuildFilter_TimeRangeCondition(t *testing.T) { + startTime := time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC) + endTime := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + + filters := &FilterSet{ + Must: &ConditionSet{ + Conditions: []FilterCondition{ + TimeRangeCondition{ + Key: "created_at", + Value: TimeRange{ + Gte: &startTime, + Lt: &endTime, + }, + }, + }, + }, + } + result := buildFilter(filters) + + if result == nil { + t.Fatal("expected filter, got nil") + } + if len(result.Must) != 1 { + t.Errorf("expected 1 Must condition, got %d", len(result.Must)) + } +} + +func TestBuildFilter_TimeRangeAllBounds(t *testing.T) { + gt := time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC) + gte := time.Date(2023, 2, 1, 0, 0, 0, 0, time.UTC) + lt := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + lte := time.Date(2024, 2, 1, 0, 0, 0, 0, time.UTC) + + filters := &FilterSet{ + Must: &ConditionSet{ + Conditions: []FilterCondition{ + TimeRangeCondition{ + Key: "updated_at", + Value: TimeRange{ + Gt: >, + Gte: >e, + Lt: <, + Lte: <e, + }, + }, + }, + }, + } + result := buildFilter(filters) + + if result == nil { + t.Fatal("expected filter, got nil") + } + if len(result.Must) != 1 { + t.Errorf("expected 1 Must condition, got %d", len(result.Must)) + } +} + +func TestBuildFilter_EmptyTimeRange(t *testing.T) { + filters := &FilterSet{ + Must: &ConditionSet{ + Conditions: []FilterCondition{ + TimeRangeCondition{ + Key: "created_at", + Value: TimeRange{}, // All nil + }, + }, + }, + } + result := buildFilter(filters) + + // Empty TimeRange returns nil condition, so filter should be nil + if result != nil { + t.Errorf("expected nil for empty time range, got %v", result) + } +} + +func TestBuildConditions_NilConditionSet(t *testing.T) { + result := buildConditions(nil) + if result != nil { + t.Errorf("expected nil, got %v", result) + } +} + +func TestTextCondition_ToQdrantCondition(t *testing.T) { + c := TextCondition{Key: "city", Value: "London"} + result := c.toQdrantCondition() + + if len(result) != 1 { + t.Errorf("expected 1 condition, got %d", len(result)) + } +} + +func TestBoolCondition_ToQdrantCondition(t *testing.T) { + c := BoolCondition{Key: "active", Value: true} + result := c.toQdrantCondition() + + if len(result) != 1 { + t.Errorf("expected 1 condition, got %d", len(result)) + } +} + +func TestIntCondition_ToQdrantCondition(t *testing.T) { + c := IntCondition{Key: "priority", Value: 42} + result := c.toQdrantCondition() + + if len(result) != 1 { + t.Errorf("expected 1 condition, got %d", len(result)) + } +} + +func TestTimeRangeCondition_ToQdrantCondition(t *testing.T) { + now := time.Now() + c := TimeRangeCondition{ + Key: "created_at", + Value: TimeRange{Gte: &now}, + } + result := c.toQdrantCondition() + + if len(result) != 1 { + t.Errorf("expected 1 condition, got %d", len(result)) + } +} + +func TestTimeRangeCondition_EmptyRange(t *testing.T) { + c := TimeRangeCondition{ + Key: "created_at", + Value: TimeRange{}, // All nil + } + result := c.toQdrantCondition() + + if result != nil { + t.Errorf("expected nil for empty time range, got %v", result) + } +} + +func TestToTimestamp_Nil(t *testing.T) { + result := toTimestamp(nil) + if result != nil { + t.Errorf("expected nil, got %v", result) + } +} + +func TestToTimestamp_ValidTime(t *testing.T) { + now := time.Now() + result := toTimestamp(&now) + + if result == nil { + t.Fatal("expected timestamp, got nil") + } + if result.AsTime().Unix() != now.Unix() { + t.Errorf("timestamp mismatch: expected %v, got %v", now.Unix(), result.AsTime().Unix()) + } +} + +// === FieldType Tests === + +func TestResolveFieldKey_InternalField(t *testing.T) { + key := resolveFieldKey("search_store_id", InternalField) + expected := "search_store_id" + if key != expected { + t.Errorf("expected %q, got %q", expected, key) + } +} + +func TestResolveFieldKey_UserField(t *testing.T) { + key := resolveFieldKey("document_id", UserField) + expected := "custom.document_id" + if key != expected { + t.Errorf("expected %q, got %q", expected, key) + } +} + +func TestResolveFieldKey_UserField_PreventDoublePrefix(t *testing.T) { + // If key already has prefix, don't add again + key := resolveFieldKey("custom.document_id", UserField) + expected := "custom.document_id" + if key != expected { + t.Errorf("expected %q, got %q (double prefix detected)", expected, key) + } +} + +func TestTextCondition_InternalField(t *testing.T) { + c := TextCondition{Key: "search_store_id", Value: "store-123", FieldType: InternalField} + result := c.toQdrantCondition() + + if len(result) != 1 { + t.Errorf("expected 1 condition, got %d", len(result)) + } + // Internal field should NOT have prefix +} + +func TestTextCondition_UserField(t *testing.T) { + c := TextCondition{Key: "document_id", Value: "doc-456", FieldType: UserField} + result := c.toQdrantCondition() + + if len(result) != 1 { + t.Errorf("expected 1 condition, got %d", len(result)) + } + // User field should have "custom." prefix +} + +func TestBoolCondition_UserField(t *testing.T) { + c := BoolCondition{Key: "is_reviewed", Value: true, FieldType: UserField} + result := c.toQdrantCondition() + + if len(result) != 1 { + t.Errorf("expected 1 condition, got %d", len(result)) + } +} + +func TestIntCondition_UserField(t *testing.T) { + c := IntCondition{Key: "version", Value: 2, FieldType: UserField} + result := c.toQdrantCondition() + + if len(result) != 1 { + t.Errorf("expected 1 condition, got %d", len(result)) + } +} + +func TestTimeRangeCondition_UserField(t *testing.T) { + now := time.Now() + c := TimeRangeCondition{ + Key: "uploaded_at", + Value: TimeRange{Gte: &now}, + FieldType: UserField, + } + result := c.toQdrantCondition() + + if len(result) != 1 { + t.Errorf("expected 1 condition, got %d", len(result)) + } +} + +func TestBuildFilter_MixedInternalAndUserFields(t *testing.T) { + // search_store_id = "store-123" (internal) AND custom.category = "reports" (user) + filters := &FilterSet{ + Must: &ConditionSet{ + Conditions: []FilterCondition{ + TextCondition{Key: "search_store_id", Value: "store-123", FieldType: InternalField}, + TextCondition{Key: "category", Value: "reports", FieldType: UserField}, + BoolCondition{Key: "is_published", Value: true, FieldType: UserField}, + }, + }, + } + result := buildFilter(filters) + + if result == nil { + t.Fatal("expected filter, got nil") + } + if len(result.Must) != 3 { + t.Errorf("expected 3 Must conditions, got %d", len(result.Must)) + } +} + +// === BuildPayload Tests === + +func TestBuildPayload_OnlyInternal(t *testing.T) { + internal := map[string]any{ + "search_store_id": "store-123", + "modalities": []string{"text"}, + } + payload := BuildPayload(internal, nil) + + if payload["search_store_id"] != "store-123" { + t.Errorf("expected search_store_id at top-level") + } + if _, exists := payload["custom"]; exists { + t.Errorf("custom should not exist when user is nil") + } +} + +func TestBuildPayload_OnlyUser(t *testing.T) { + user := map[string]any{ + "document_id": "doc-456", + "author": "John", + } + payload := BuildPayload(nil, user) + + custom, ok := payload["custom"].(map[string]any) + if !ok { + t.Fatal("expected custom field") + } + if custom["document_id"] != "doc-456" { + t.Errorf("expected document_id in custom") + } + if custom["author"] != "John" { + t.Errorf("expected author in custom") + } +} + +func TestBuildPayload_BothInternalAndUser(t *testing.T) { + internal := map[string]any{ + "search_store_id": "store-123", + } + user := map[string]any{ + "document_id": "doc-456", + "category": "reports", + } + payload := BuildPayload(internal, user) + + // Check internal at top-level + if payload["search_store_id"] != "store-123" { + t.Errorf("expected search_store_id at top-level") + } + + // Check user under custom + custom, ok := payload["custom"].(map[string]any) + if !ok { + t.Fatal("expected custom field") + } + if custom["document_id"] != "doc-456" { + t.Errorf("expected document_id in custom") + } + if custom["category"] != "reports" { + t.Errorf("expected category in custom") + } +} + +func TestBuildPayload_EmptyUser(t *testing.T) { + internal := map[string]any{ + "search_store_id": "store-123", + } + user := map[string]any{} // Empty, not nil + payload := BuildPayload(internal, user) + + if _, exists := payload["custom"]; exists { + t.Errorf("custom should not exist when user is empty") + } +} + +func TestResolveFieldKey_ActualPath(t *testing.T) { + tests := []struct { + key string + fieldType FieldType + expected string + }{ + {"city", InternalField, "city"}, + {"city", UserField, "custom.city"}, + {"custom.city", UserField, "custom.city"}, // No double prefix + } + + for _, tt := range tests { + result := resolveFieldKey(tt.key, tt.fieldType) + if result != tt.expected { + t.Errorf("resolveFieldKey(%q, %v) = %q, want %q", + tt.key, tt.fieldType, result, tt.expected) + } + } +} diff --git a/v1/qdrant/utils.go b/v1/qdrant/utils.go index 7fbed26..9909631 100644 --- a/v1/qdrant/utils.go +++ b/v1/qdrant/utils.go @@ -77,16 +77,15 @@ type SearchResultInterface interface { // of the underlying database implementation. // // Fields: -// • Name — The unique name of the collection. -// • Status — Current operational state (e.g., "Green", "Yellow"). -// • VectorSize — The dimension of stored vectors (e.g., 1536). -// • Distance — The similarity metric used ("Cosine", "Dot", "Euclid"). -// • Vectors — Total number of stored vectors in the collection. -// • Points — Total number of indexed points/documents in the collection. +// - Name — The unique name of the collection. +// - Status — Current operational state (e.g., "Green", "Yellow"). +// - VectorSize — The dimension of stored vectors (e.g., 1536). +// - Distance — The similarity metric used ("Cosine", "Dot", "Euclid"). +// - Vectors — Total number of stored vectors in the collection. +// - Points — Total number of indexed points/documents in the collection. // -// This struct serves as an abstraction layer between Qdrant’s low-level +// This struct serves as an abstraction layer between Qdrant's low-level // protobuf models and the higher-level application logic. - type Collection struct { Name string Status string @@ -101,7 +100,7 @@ type SearchRequest struct { CollectionName string Vector []float32 TopK int - Filters map[string]string //Optional: key-value filters + Filters *FilterSet // Optional: key-value filters } // validateSearchInput validates common search parameters @@ -118,31 +117,6 @@ func validateSearchInput(collectionName string, vector []float32, topK int) erro return nil } -// buildFilter constructs a Qdrant filter from key-value pairs (AND logic) -func buildFilter(filters map[string]string) *qdrant.Filter { - if len(filters) == 0 { - return nil - } - - conditions := make([]*qdrant.Condition, 0, len(filters)) - for key, value := range filters { - conditions = append(conditions, &qdrant.Condition{ - ConditionOneOf: &qdrant.Condition_Field{ - Field: &qdrant.FieldCondition{ - Key: key, - Match: &qdrant.Match{ - MatchValue: &qdrant.Match_Keyword{ - Keyword: value, - }, - }, - }, - }, - }) - } - - return &qdrant.Filter{Must: conditions} -} - // extractVectorDetails ────────────────────────────────────────────────────────────── // extractVectorDetails // ────────────────────────────────────────────────────────────── @@ -152,12 +126,12 @@ func buildFilter(filters map[string]string) *qdrant.Filter { // `CollectionInfo` object. // // Qdrant represents vector configuration data using a deeply nested protobuf -// structure with “oneof” wrappers. This helper navigates that hierarchy, +// structure with "oneof" wrappers. This helper navigates that hierarchy, // performs type assertions, and guards against nil pointer dereferences. // // It returns: -// • int — vector dimension (size of embedding vectors) -// • string — distance metric used for similarity search +// - int — vector dimension (size of embedding vectors) +// - string — distance metric used for similarity search // // If any nested field is missing or of an unexpected type, the function // gracefully returns default values (0, ""). @@ -166,7 +140,6 @@ func buildFilter(filters map[string]string) *qdrant.Filter { // // size, distance := extractVectorDetails(info) // log.Printf("Vector size=%d, distance=%s", size, distance) - func extractVectorDetails(info *qdrant.CollectionInfo) (int, string) { if info == nil || info.Config == nil || @@ -191,4 +164,3 @@ func derefUint64(v *uint64) uint64 { } return 0 } - From 1d2243168903c58b64a5677e37fb05ff77b05104 Mon Sep 17 00:00:00 2001 From: Shinu Joseph Date: Mon, 8 Dec 2025 10:52:11 +0100 Subject: [PATCH 2/5] chore: export ToQdrantCondition method --- v1/qdrant/filters.go | 12 ++++++------ v1/qdrant/filters_test.go | 20 ++++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/v1/qdrant/filters.go b/v1/qdrant/filters.go index be04866..1e3b9c7 100644 --- a/v1/qdrant/filters.go +++ b/v1/qdrant/filters.go @@ -13,7 +13,7 @@ const UserPayloadPrefix = "custom" // FilterCondition is the interface for all filter conditions type FilterCondition interface { - toQdrantCondition() []*qdrant.Condition + ToQdrantCondition() []*qdrant.Condition } // FieldType indicates whether a field is internal or user-defined @@ -40,7 +40,7 @@ type MatchCondition[T comparable] struct { FieldType FieldType // Internal or User field (default: InternalField) } -func (c MatchCondition[T]) toQdrantCondition() []*qdrant.Condition { +func (c MatchCondition[T]) ToQdrantCondition() []*qdrant.Condition { key := resolveFieldKey(c.Key, c.FieldType) switch v := any(c.Value).(type) { case string: @@ -64,7 +64,7 @@ type MatchAnyCondition[T string | int64] struct { FieldType FieldType } -func (c MatchAnyCondition[T]) toQdrantCondition() []*qdrant.Condition { +func (c MatchAnyCondition[T]) ToQdrantCondition() []*qdrant.Condition { key := resolveFieldKey(c.Key, c.FieldType) switch v := any(c.Values).(type) { case []string: @@ -84,7 +84,7 @@ type MatchExceptCondition[T string | int64] struct { FieldType FieldType } -func (c MatchExceptCondition[T]) toQdrantCondition() []*qdrant.Condition { +func (c MatchExceptCondition[T]) ToQdrantCondition() []*qdrant.Condition { key := resolveFieldKey(c.Key, c.FieldType) switch v := any(c.Values).(type) { case []string: @@ -111,7 +111,7 @@ type TimeRangeCondition struct { FieldType FieldType // Internal or User field (default: InternalField) } -func (c TimeRangeCondition) toQdrantCondition() []*qdrant.Condition { +func (c TimeRangeCondition) ToQdrantCondition() []*qdrant.Condition { return buildDateTimeRangeConditions(resolveFieldKey(c.Key, c.FieldType), c.Value) } @@ -188,7 +188,7 @@ func buildConditions(cs *ConditionSet) []*qdrant.Condition { var conditions []*qdrant.Condition for _, c := range cs.Conditions { - conditions = append(conditions, c.toQdrantCondition()...) + conditions = append(conditions, c.ToQdrantCondition()...) } return conditions } diff --git a/v1/qdrant/filters_test.go b/v1/qdrant/filters_test.go index 3812968..49832be 100644 --- a/v1/qdrant/filters_test.go +++ b/v1/qdrant/filters_test.go @@ -265,7 +265,7 @@ func TestBuildConditions_NilConditionSet(t *testing.T) { func TestTextCondition_ToQdrantCondition(t *testing.T) { c := TextCondition{Key: "city", Value: "London"} - result := c.toQdrantCondition() + result := c.ToQdrantCondition() if len(result) != 1 { t.Errorf("expected 1 condition, got %d", len(result)) @@ -274,7 +274,7 @@ func TestTextCondition_ToQdrantCondition(t *testing.T) { func TestBoolCondition_ToQdrantCondition(t *testing.T) { c := BoolCondition{Key: "active", Value: true} - result := c.toQdrantCondition() + result := c.ToQdrantCondition() if len(result) != 1 { t.Errorf("expected 1 condition, got %d", len(result)) @@ -283,7 +283,7 @@ func TestBoolCondition_ToQdrantCondition(t *testing.T) { func TestIntCondition_ToQdrantCondition(t *testing.T) { c := IntCondition{Key: "priority", Value: 42} - result := c.toQdrantCondition() + result := c.ToQdrantCondition() if len(result) != 1 { t.Errorf("expected 1 condition, got %d", len(result)) @@ -296,7 +296,7 @@ func TestTimeRangeCondition_ToQdrantCondition(t *testing.T) { Key: "created_at", Value: TimeRange{Gte: &now}, } - result := c.toQdrantCondition() + result := c.ToQdrantCondition() if len(result) != 1 { t.Errorf("expected 1 condition, got %d", len(result)) @@ -308,7 +308,7 @@ func TestTimeRangeCondition_EmptyRange(t *testing.T) { Key: "created_at", Value: TimeRange{}, // All nil } - result := c.toQdrantCondition() + result := c.ToQdrantCondition() if result != nil { t.Errorf("expected nil for empty time range, got %v", result) @@ -363,7 +363,7 @@ func TestResolveFieldKey_UserField_PreventDoublePrefix(t *testing.T) { func TestTextCondition_InternalField(t *testing.T) { c := TextCondition{Key: "search_store_id", Value: "store-123", FieldType: InternalField} - result := c.toQdrantCondition() + result := c.ToQdrantCondition() if len(result) != 1 { t.Errorf("expected 1 condition, got %d", len(result)) @@ -373,7 +373,7 @@ func TestTextCondition_InternalField(t *testing.T) { func TestTextCondition_UserField(t *testing.T) { c := TextCondition{Key: "document_id", Value: "doc-456", FieldType: UserField} - result := c.toQdrantCondition() + result := c.ToQdrantCondition() if len(result) != 1 { t.Errorf("expected 1 condition, got %d", len(result)) @@ -383,7 +383,7 @@ func TestTextCondition_UserField(t *testing.T) { func TestBoolCondition_UserField(t *testing.T) { c := BoolCondition{Key: "is_reviewed", Value: true, FieldType: UserField} - result := c.toQdrantCondition() + result := c.ToQdrantCondition() if len(result) != 1 { t.Errorf("expected 1 condition, got %d", len(result)) @@ -392,7 +392,7 @@ func TestBoolCondition_UserField(t *testing.T) { func TestIntCondition_UserField(t *testing.T) { c := IntCondition{Key: "version", Value: 2, FieldType: UserField} - result := c.toQdrantCondition() + result := c.ToQdrantCondition() if len(result) != 1 { t.Errorf("expected 1 condition, got %d", len(result)) @@ -406,7 +406,7 @@ func TestTimeRangeCondition_UserField(t *testing.T) { Value: TimeRange{Gte: &now}, FieldType: UserField, } - result := c.toQdrantCondition() + result := c.ToQdrantCondition() if len(result) != 1 { t.Errorf("expected 1 condition, got %d", len(result)) From 5b8ef2852a4eb8509f6193ef2df3c83cad68c84d Mon Sep 17 00:00:00 2001 From: Shinu Joseph Date: Mon, 8 Dec 2025 13:46:22 +0100 Subject: [PATCH 3/5] feat: add support for numeric ranges, checks for null and empty fields --- v1/qdrant/filters.go | 147 +++++++++++--- v1/qdrant/filters_test.go | 393 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 513 insertions(+), 27 deletions(-) diff --git a/v1/qdrant/filters.go b/v1/qdrant/filters.go index 1e3b9c7..7eb2568 100644 --- a/v1/qdrant/filters.go +++ b/v1/qdrant/filters.go @@ -1,6 +1,7 @@ package qdrant import ( + "encoding/json" "strings" "time" @@ -28,16 +29,24 @@ const ( // TimeRange represents a time-based filter condition type TimeRange struct { - Gt *time.Time // Greater than this time - Gte *time.Time // Greater than or equal to this time - Lt *time.Time // Less than this time - Lte *time.Time // Less than or equal to this time + Gt *time.Time `json:"after,omitempty"` // Greater than this time + Gte *time.Time `json:"atOrAfter,omitempty"` // Greater than or equal to this time + Lt *time.Time `json:"before,omitempty"` // Less than this time + Lte *time.Time `json:"atOrBefore,omitempty"` // Less than or equal to this time +} + +// NumericRange represents a numeric range filter condition +type NumericRange struct { + Gt *float64 `json:"greaterThan,omitempty"` // Greater than + Gte *float64 `json:"greaterThanOrEqualTo,omitempty"` // Greater than or equal + Lt *float64 `json:"lessThan,omitempty"` // Less than + Lte *float64 `json:"lessThanOrEqualTo,omitempty"` // Less than or equal } type MatchCondition[T comparable] struct { - Key string - Value T - FieldType FieldType // Internal or User field (default: InternalField) + Key string `json:"field"` + Value T `json:"equalTo"` + FieldType FieldType `json:"-"` // Internal or User field (default: InternalField) } func (c MatchCondition[T]) ToQdrantCondition() []*qdrant.Condition { @@ -59,9 +68,9 @@ func (c MatchCondition[T]) ToQdrantCondition() []*qdrant.Condition { // MatchAnyCondition matches if value is one of the given values (IN operator) // Applicable to keyword (string) and integer payloads type MatchAnyCondition[T string | int64] struct { - Key string - Values []T - FieldType FieldType + Key string `json:"field"` + Values []T `json:"anyOf"` + FieldType FieldType `json:"-"` // Internal or User field (default: InternalField) } func (c MatchAnyCondition[T]) ToQdrantCondition() []*qdrant.Condition { @@ -79,9 +88,9 @@ func (c MatchAnyCondition[T]) ToQdrantCondition() []*qdrant.Condition { // MatchExceptCondition matches if value is NOT one of the given values (NOT IN operator) // Applicable to keyword (string) and integer payloads type MatchExceptCondition[T string | int64] struct { - Key string - Values []T - FieldType FieldType + Key string `json:"field"` + Values []T `json:"noneOf"` + FieldType FieldType `json:"-"` // Internal or User field (default: InternalField) } func (c MatchExceptCondition[T]) ToQdrantCondition() []*qdrant.Condition { @@ -96,25 +105,70 @@ func (c MatchExceptCondition[T]) ToQdrantCondition() []*qdrant.Condition { } } -type TextCondition = MatchCondition[string] -type BoolCondition = MatchCondition[bool] -type IntCondition = MatchCondition[int64] -type TextAnyCondition = MatchAnyCondition[string] -type IntAnyCondition = MatchAnyCondition[int64] -type TextExceptCondition = MatchExceptCondition[string] -type IntExceptCondition = MatchExceptCondition[int64] +type TextCondition = MatchCondition[string] // Exact string match +type BoolCondition = MatchCondition[bool] // Exact boolean match +type IntCondition = MatchCondition[int64] // Exact integer match +type TextAnyCondition = MatchAnyCondition[string] // String IN operator +type IntAnyCondition = MatchAnyCondition[int64] // Integer IN operator +type TextExceptCondition = MatchExceptCondition[string] // String NOT IN +type IntExceptCondition = MatchExceptCondition[int64] // Integer NOT IN // TimeRangeCondition represents a time range filter condition type TimeRangeCondition struct { - Key string - Value TimeRange - FieldType FieldType // Internal or User field (default: InternalField) + Key string `json:"field"` + Value TimeRange `json:"-"` + FieldType FieldType `json:"-"` } func (c TimeRangeCondition) ToQdrantCondition() []*qdrant.Condition { return buildDateTimeRangeConditions(resolveFieldKey(c.Key, c.FieldType), c.Value) } +func (c TimeRangeCondition) MarshalJSON() ([]byte, error) { + type Alias struct { + Field string `json:"field"` + After *time.Time `json:"after,omitempty"` + AtOrAfter *time.Time `json:"atOrAfter,omitempty"` + Before *time.Time `json:"before,omitempty"` + AtOrBefore *time.Time `json:"atOrBefore,omitempty"` + } + return json.Marshal(Alias{ + Field: c.Key, + After: c.Value.Gt, + AtOrAfter: c.Value.Gte, + Before: c.Value.Lt, + AtOrBefore: c.Value.Lte, + }) +} + +// NumericRangeCondition represents a numeric range filter +type NumericRangeCondition struct { + Key string `json:"field"` + Value NumericRange `json:"-"` + FieldType FieldType `json:"-"` +} + +func (c NumericRangeCondition) ToQdrantCondition() []*qdrant.Condition { + return buildNumericRangeConditions(resolveFieldKey(c.Key, c.FieldType), c.Value) +} + +func (c NumericRangeCondition) MarshalJSON() ([]byte, error) { + type Alias struct { + Field string `json:"field"` + GreaterThan *float64 `json:"greaterThan,omitempty"` + GreaterThanOrEqualTo *float64 `json:"greaterThanOrEqualTo,omitempty"` + LessThan *float64 `json:"lessThan,omitempty"` + LessThanOrEqualTo *float64 `json:"lessThanOrEqualTo,omitempty"` + } + return json.Marshal(Alias{ + Field: c.Key, + GreaterThan: c.Value.Gt, + GreaterThanOrEqualTo: c.Value.Gte, + LessThan: c.Value.Lt, + LessThanOrEqualTo: c.Value.Lte, + }) +} + // resolveFieldKey returns the full field path based on FieldType // Internal fields: "search_store_id" -> "search_store_id" // User fields: "document_id" -> "custom.document_id" @@ -131,7 +185,7 @@ func resolveFieldKey(key string, fieldType FieldType) string { // ConditionSet holds conditions for a single clause type ConditionSet struct { - Conditions []FilterCondition + Conditions []FilterCondition `json:"conditions,omitempty"` } // FilterSet supports Must (AND), Should (OR), and MustNot (NOT) clauses. @@ -147,9 +201,9 @@ type ConditionSet struct { // }, // } type FilterSet struct { - Must *ConditionSet // AND - all conditions must match - Should *ConditionSet // OR - at least one condition must match - MustNot *ConditionSet // NOT - none of the conditions should match + Must *ConditionSet `json:"with,omitempty"` // AND - all conditions must match + Should *ConditionSet `json:"withOneOf,omitempty"` // OR - at least one condition must match + MustNot *ConditionSet `json:"without,omitempty"` // NOT - none of the conditions should match } // buildFilter constructs a Qdrant filter from FilterSet @@ -210,6 +264,23 @@ func buildDateTimeRangeConditions(key string, tr TimeRange) []*qdrant.Condition return []*qdrant.Condition{qdrant.NewDatetimeRange(key, dateRange)} } +// buildNumericRangeConditions creates numeric range conditions +func buildNumericRangeConditions(key string, nr NumericRange) []*qdrant.Condition { + rangeFilter := &qdrant.Range{ + Gt: nr.Gt, + Gte: nr.Gte, + Lt: nr.Lt, + Lte: nr.Lte, + } + + // Check if any field is set + if rangeFilter.Gt == nil && rangeFilter.Gte == nil && rangeFilter.Lt == nil && rangeFilter.Lte == nil { + return nil + } + + return []*qdrant.Condition{qdrant.NewRange(key, rangeFilter)} +} + // toTimestamp converts a *time.Time to *timestamppb.Timestamp (nil-safe) func toTimestamp(t *time.Time) *timestamppb.Timestamp { if t == nil { @@ -218,6 +289,28 @@ func toTimestamp(t *time.Time) *timestamppb.Timestamp { return timestamppb.New(*t) } +// IsNullCondition checks if a field is null +type IsNullCondition struct { + Key string `json:"field"` + FieldType FieldType `json:"-"` // Internal or User field (default: InternalField) +} + +func (c IsNullCondition) ToQdrantCondition() []*qdrant.Condition { + key := resolveFieldKey(c.Key, c.FieldType) + return []*qdrant.Condition{qdrant.NewIsNull(key)} +} + +// IsEmptyCondition checks if a field is empty (does not exist, null, or []) +type IsEmptyCondition struct { + Key string `json:"field"` + FieldType FieldType `json:"-"` // Internal or User field (default: InternalField) +} + +func (c IsEmptyCondition) ToQdrantCondition() []*qdrant.Condition { + key := resolveFieldKey(c.Key, c.FieldType) + return []*qdrant.Condition{qdrant.NewIsEmpty(key)} +} + // === Payload Helpers === // BuildPayload creates a Qdrant payload with separated internal and user fields diff --git a/v1/qdrant/filters_test.go b/v1/qdrant/filters_test.go index 49832be..b373b01 100644 --- a/v1/qdrant/filters_test.go +++ b/v1/qdrant/filters_test.go @@ -529,3 +529,396 @@ func TestResolveFieldKey_ActualPath(t *testing.T) { } } } + +// === MatchAnyCondition Tests === + +func TestTextAnyCondition_ToQdrantCondition(t *testing.T) { + c := TextAnyCondition{Key: "city", Values: []string{"London", "Berlin", "Paris"}} + result := c.ToQdrantCondition() + + if len(result) != 1 { + t.Errorf("expected 1 condition, got %d", len(result)) + } +} + +func TestIntAnyCondition_ToQdrantCondition(t *testing.T) { + c := IntAnyCondition{Key: "priority", Values: []int64{1, 2, 3}} + result := c.ToQdrantCondition() + + if len(result) != 1 { + t.Errorf("expected 1 condition, got %d", len(result)) + } +} + +func TestTextAnyCondition_UserField(t *testing.T) { + c := TextAnyCondition{Key: "category", Values: []string{"tech", "science"}, FieldType: UserField} + result := c.ToQdrantCondition() + + if len(result) != 1 { + t.Errorf("expected 1 condition, got %d", len(result)) + } +} + +func TestBuildFilter_WithTextAnyCondition(t *testing.T) { + // city IN ("London", "Berlin") + filters := &FilterSet{ + Must: &ConditionSet{ + Conditions: []FilterCondition{ + TextAnyCondition{Key: "city", Values: []string{"London", "Berlin"}}, + }, + }, + } + result := buildFilter(filters) + + if result == nil { + t.Fatal("expected filter, got nil") + } + if len(result.Must) != 1 { + t.Errorf("expected 1 Must condition, got %d", len(result.Must)) + } +} + +// === MatchExceptCondition Tests === + +func TestTextExceptCondition_ToQdrantCondition(t *testing.T) { + c := TextExceptCondition{Key: "city", Values: []string{"Paris", "Madrid"}} + result := c.ToQdrantCondition() + + if len(result) != 1 { + t.Errorf("expected 1 condition, got %d", len(result)) + } +} + +func TestIntExceptCondition_ToQdrantCondition(t *testing.T) { + c := IntExceptCondition{Key: "priority", Values: []int64{0, -1}} + result := c.ToQdrantCondition() + + if len(result) != 1 { + t.Errorf("expected 1 condition, got %d", len(result)) + } +} + +func TestTextExceptCondition_UserField(t *testing.T) { + c := TextExceptCondition{Key: "status", Values: []string{"draft", "deleted"}, FieldType: UserField} + result := c.ToQdrantCondition() + + if len(result) != 1 { + t.Errorf("expected 1 condition, got %d", len(result)) + } +} + +func TestBuildFilter_WithTextExceptCondition(t *testing.T) { + // city NOT IN ("Paris", "Madrid") + filters := &FilterSet{ + MustNot: &ConditionSet{ + Conditions: []FilterCondition{ + TextExceptCondition{Key: "city", Values: []string{"Paris", "Madrid"}}, + }, + }, + } + result := buildFilter(filters) + + if result == nil { + t.Fatal("expected filter, got nil") + } + if len(result.MustNot) != 1 { + t.Errorf("expected 1 MustNot condition, got %d", len(result.MustNot)) + } +} + +// === NumericRangeCondition Tests === + +func TestNumericRangeCondition_ToQdrantCondition(t *testing.T) { + minPrice := 100.0 + maxPrice := 500.0 + c := NumericRangeCondition{ + Key: "price", + Value: NumericRange{ + Gte: &minPrice, + Lte: &maxPrice, + }, + } + result := c.ToQdrantCondition() + + if len(result) != 1 { + t.Errorf("expected 1 condition, got %d", len(result)) + } +} + +func TestNumericRangeCondition_AllBounds(t *testing.T) { + gt := 10.0 + gte := 20.0 + lt := 100.0 + lte := 90.0 + c := NumericRangeCondition{ + Key: "score", + Value: NumericRange{ + Gt: >, + Gte: >e, + Lt: <, + Lte: <e, + }, + } + result := c.ToQdrantCondition() + + if len(result) != 1 { + t.Errorf("expected 1 condition, got %d", len(result)) + } +} + +func TestNumericRangeCondition_EmptyRange(t *testing.T) { + c := NumericRangeCondition{ + Key: "price", + Value: NumericRange{}, // All nil + } + result := c.ToQdrantCondition() + + if result != nil { + t.Errorf("expected nil for empty numeric range, got %v", result) + } +} + +func TestNumericRangeCondition_UserField(t *testing.T) { + minPrice := 50.0 + c := NumericRangeCondition{ + Key: "price", + Value: NumericRange{Gte: &minPrice}, + FieldType: UserField, + } + result := c.ToQdrantCondition() + + if len(result) != 1 { + t.Errorf("expected 1 condition, got %d", len(result)) + } +} + +func TestBuildFilter_WithNumericRangeCondition(t *testing.T) { + minPrice := 100.0 + maxPrice := 500.0 + filters := &FilterSet{ + Must: &ConditionSet{ + Conditions: []FilterCondition{ + NumericRangeCondition{ + Key: "price", + Value: NumericRange{ + Gte: &minPrice, + Lte: &maxPrice, + }, + }, + }, + }, + } + result := buildFilter(filters) + + if result == nil { + t.Fatal("expected filter, got nil") + } + if len(result.Must) != 1 { + t.Errorf("expected 1 Must condition, got %d", len(result.Must)) + } +} + +// === IsNullCondition Tests === + +func TestIsNullCondition_ToQdrantCondition(t *testing.T) { + c := IsNullCondition{Key: "deleted_at"} + result := c.ToQdrantCondition() + + if len(result) != 1 { + t.Errorf("expected 1 condition, got %d", len(result)) + } +} + +func TestIsNullCondition_UserField(t *testing.T) { + c := IsNullCondition{Key: "review_date", FieldType: UserField} + result := c.ToQdrantCondition() + + if len(result) != 1 { + t.Errorf("expected 1 condition, got %d", len(result)) + } +} + +func TestBuildFilter_WithIsNullCondition(t *testing.T) { + filters := &FilterSet{ + Must: &ConditionSet{ + Conditions: []FilterCondition{ + IsNullCondition{Key: "deleted_at"}, + }, + }, + } + result := buildFilter(filters) + + if result == nil { + t.Fatal("expected filter, got nil") + } + if len(result.Must) != 1 { + t.Errorf("expected 1 Must condition, got %d", len(result.Must)) + } +} + +// === IsEmptyCondition Tests === + +func TestIsEmptyCondition_ToQdrantCondition(t *testing.T) { + c := IsEmptyCondition{Key: "tags"} + result := c.ToQdrantCondition() + + if len(result) != 1 { + t.Errorf("expected 1 condition, got %d", len(result)) + } +} + +func TestIsEmptyCondition_UserField(t *testing.T) { + c := IsEmptyCondition{Key: "categories", FieldType: UserField} + result := c.ToQdrantCondition() + + if len(result) != 1 { + t.Errorf("expected 1 condition, got %d", len(result)) + } +} + +func TestBuildFilter_WithIsEmptyCondition(t *testing.T) { + // Find documents where tags is NOT empty (using MustNot) + filters := &FilterSet{ + MustNot: &ConditionSet{ + Conditions: []FilterCondition{ + IsEmptyCondition{Key: "tags"}, + }, + }, + } + result := buildFilter(filters) + + if result == nil { + t.Fatal("expected filter, got nil") + } + if len(result.MustNot) != 1 { + t.Errorf("expected 1 MustNot condition, got %d", len(result.MustNot)) + } +} + +// === MarshalJSON Tests === + +func TestTimeRangeCondition_MarshalJSON(t *testing.T) { + startTime := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + endTime := time.Date(2024, 12, 31, 0, 0, 0, 0, time.UTC) + + c := TimeRangeCondition{ + Key: "created_at", + Value: TimeRange{ + Gte: &startTime, + Lt: &endTime, + }, + } + + data, err := c.MarshalJSON() + if err != nil { + t.Fatalf("MarshalJSON failed: %v", err) + } + + jsonStr := string(data) + // Check that it contains expected fields + if !contains(jsonStr, `"field":"created_at"`) { + t.Errorf("expected field in JSON, got %s", jsonStr) + } + if !contains(jsonStr, `"atOrAfter"`) { + t.Errorf("expected atOrAfter in JSON, got %s", jsonStr) + } + if !contains(jsonStr, `"before"`) { + t.Errorf("expected before in JSON, got %s", jsonStr) + } +} + +func TestNumericRangeCondition_MarshalJSON(t *testing.T) { + minPrice := 100.0 + maxPrice := 500.0 + + c := NumericRangeCondition{ + Key: "price", + Value: NumericRange{ + Gte: &minPrice, + Lte: &maxPrice, + }, + } + + data, err := c.MarshalJSON() + if err != nil { + t.Fatalf("MarshalJSON failed: %v", err) + } + + jsonStr := string(data) + if !contains(jsonStr, `"field":"price"`) { + t.Errorf("expected field in JSON, got %s", jsonStr) + } + if !contains(jsonStr, `"greaterThanOrEqualTo"`) { + t.Errorf("expected greaterThanOrEqualTo in JSON, got %s", jsonStr) + } + if !contains(jsonStr, `"lessThanOrEqualTo"`) { + t.Errorf("expected lessThanOrEqualTo in JSON, got %s", jsonStr) + } +} + +// === Complex Filter Tests === + +func TestBuildFilter_ComplexCombination(t *testing.T) { + startTime := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + minPrice := 100.0 + + // Complex filter: + // (search_store_id = "store-123" AND created_at >= 2024-01-01 AND price >= 100) + // AND (city IN ("London", "Berlin")) + // AND NOT (deleted = true OR status IN ("draft", "archived")) + filters := &FilterSet{ + Must: &ConditionSet{ + Conditions: []FilterCondition{ + TextCondition{Key: "search_store_id", Value: "store-123"}, + TimeRangeCondition{ + Key: "created_at", + Value: TimeRange{Gte: &startTime}, + }, + NumericRangeCondition{ + Key: "price", + Value: NumericRange{Gte: &minPrice}, + FieldType: UserField, + }, + }, + }, + Should: &ConditionSet{ + Conditions: []FilterCondition{ + TextAnyCondition{Key: "city", Values: []string{"London", "Berlin"}}, + }, + }, + MustNot: &ConditionSet{ + Conditions: []FilterCondition{ + BoolCondition{Key: "deleted", Value: true}, + TextAnyCondition{Key: "status", Values: []string{"draft", "archived"}}, + }, + }, + } + result := buildFilter(filters) + + if result == nil { + t.Fatal("expected filter, got nil") + } + if len(result.Must) != 3 { + t.Errorf("expected 3 Must conditions, got %d", len(result.Must)) + } + if len(result.Should) != 1 { + t.Errorf("expected 1 Should condition, got %d", len(result.Should)) + } + if len(result.MustNot) != 2 { + t.Errorf("expected 2 MustNot conditions, got %d", len(result.MustNot)) + } +} + +// Helper function for string contains check +func contains(s, substr string) bool { + return len(s) >= len(substr) && (s == substr || len(s) > 0 && containsHelper(s, substr)) +} + +func containsHelper(s, substr string) bool { + for i := 0; i <= len(s)-len(substr); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false +} From f1f01d08f13beaaff234d2a899d9fc55e672d218 Mon Sep 17 00:00:00 2001 From: Sounak Pradhan Date: Mon, 8 Dec 2025 16:18:23 +0100 Subject: [PATCH 4/5] feat: enhance filtering capabilities with nil condition handling and JSON unmarshalling for time and numeric range conditions --- v1/qdrant/filters.go | 83 ++++++++++++-- v1/qdrant/filters_test.go | 224 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 299 insertions(+), 8 deletions(-) diff --git a/v1/qdrant/filters.go b/v1/qdrant/filters.go index 7eb2568..a4a3d9b 100644 --- a/v1/qdrant/filters.go +++ b/v1/qdrant/filters.go @@ -43,12 +43,18 @@ type NumericRange struct { Lte *float64 `json:"lessThanOrEqualTo,omitempty"` // Less than or equal } +// MatchCondition represents an exact match condition for a field value. +// Supports string, bool, and int64 types. The FieldType defaults to InternalField +// if not specified, meaning the field is stored at the top level of the payload. +// Use UserField to indicate the field is stored under the "custom." prefix. type MatchCondition[T comparable] struct { Key string `json:"field"` Value T `json:"equalTo"` FieldType FieldType `json:"-"` // Internal or User field (default: InternalField) } +// ToQdrantCondition converts the MatchCondition to Qdrant conditions. +// Supports string, bool, and int64 types. Returns nil for unsupported types. func (c MatchCondition[T]) ToQdrantCondition() []*qdrant.Condition { key := resolveFieldKey(c.Key, c.FieldType) switch v := any(c.Value).(type) { @@ -59,14 +65,14 @@ func (c MatchCondition[T]) ToQdrantCondition() []*qdrant.Condition { case int64: return []*qdrant.Condition{qdrant.NewMatchInt(key, v)} default: - //Unsupported type + // Unsupported type - returns nil return nil } - } -// MatchAnyCondition matches if value is one of the given values (IN operator) -// Applicable to keyword (string) and integer payloads +// MatchAnyCondition matches if value is one of the given values (IN operator). +// Applicable to keyword (string) and integer payloads. +// Returns nil if Values is empty. The FieldType defaults to InternalField if not specified. type MatchAnyCondition[T string | int64] struct { Key string `json:"field"` Values []T `json:"anyOf"` @@ -74,6 +80,9 @@ type MatchAnyCondition[T string | int64] struct { } func (c MatchAnyCondition[T]) ToQdrantCondition() []*qdrant.Condition { + if len(c.Values) == 0 { + return nil + } key := resolveFieldKey(c.Key, c.FieldType) switch v := any(c.Values).(type) { case []string: @@ -85,8 +94,9 @@ func (c MatchAnyCondition[T]) ToQdrantCondition() []*qdrant.Condition { } } -// MatchExceptCondition matches if value is NOT one of the given values (NOT IN operator) -// Applicable to keyword (string) and integer payloads +// MatchExceptCondition matches if value is NOT one of the given values (NOT IN operator). +// Applicable to keyword (string) and integer payloads. +// Returns nil if Values is empty. The FieldType defaults to InternalField if not specified. type MatchExceptCondition[T string | int64] struct { Key string `json:"field"` Values []T `json:"noneOf"` @@ -94,6 +104,9 @@ type MatchExceptCondition[T string | int64] struct { } func (c MatchExceptCondition[T]) ToQdrantCondition() []*qdrant.Condition { + if len(c.Values) == 0 { + return nil + } key := resolveFieldKey(c.Key, c.FieldType) switch v := any(c.Values).(type) { case []string: @@ -141,6 +154,28 @@ func (c TimeRangeCondition) MarshalJSON() ([]byte, error) { }) } +func (c *TimeRangeCondition) UnmarshalJSON(data []byte) error { + type Alias struct { + Field string `json:"field"` + After *time.Time `json:"after,omitempty"` + AtOrAfter *time.Time `json:"atOrAfter,omitempty"` + Before *time.Time `json:"before,omitempty"` + AtOrBefore *time.Time `json:"atOrBefore,omitempty"` + } + var alias Alias + if err := json.Unmarshal(data, &alias); err != nil { + return err + } + c.Key = alias.Field + c.Value = TimeRange{ + Gt: alias.After, + Gte: alias.AtOrAfter, + Lt: alias.Before, + Lte: alias.AtOrBefore, + } + return nil +} + // NumericRangeCondition represents a numeric range filter type NumericRangeCondition struct { Key string `json:"field"` @@ -169,6 +204,28 @@ func (c NumericRangeCondition) MarshalJSON() ([]byte, error) { }) } +func (c *NumericRangeCondition) UnmarshalJSON(data []byte) error { + type Alias struct { + Field string `json:"field"` + GreaterThan *float64 `json:"greaterThan,omitempty"` + GreaterThanOrEqualTo *float64 `json:"greaterThanOrEqualTo,omitempty"` + LessThan *float64 `json:"lessThan,omitempty"` + LessThanOrEqualTo *float64 `json:"lessThanOrEqualTo,omitempty"` + } + var alias Alias + if err := json.Unmarshal(data, &alias); err != nil { + return err + } + c.Key = alias.Field + c.Value = NumericRange{ + Gt: alias.GreaterThan, + Gte: alias.GreaterThanOrEqualTo, + Lt: alias.LessThan, + Lte: alias.LessThanOrEqualTo, + } + return nil +} + // resolveFieldKey returns the full field path based on FieldType // Internal fields: "search_store_id" -> "search_store_id" // User fields: "document_id" -> "custom.document_id" @@ -235,6 +292,7 @@ func buildFilter(filters *FilterSet) *qdrant.Filter { } // buildConditions converts a ConditionSet to Qdrant conditions +// Filters out nil conditions that may be returned by invalid conditions (e.g., empty ranges) func buildConditions(cs *ConditionSet) []*qdrant.Condition { if cs == nil { return nil @@ -242,7 +300,12 @@ func buildConditions(cs *ConditionSet) []*qdrant.Condition { var conditions []*qdrant.Condition for _, c := range cs.Conditions { - conditions = append(conditions, c.ToQdrantCondition()...) + conds := c.ToQdrantCondition() + for _, cond := range conds { + if cond != nil { + conditions = append(conditions, cond) + } + } } return conditions } @@ -313,7 +376,10 @@ func (c IsEmptyCondition) ToQdrantCondition() []*qdrant.Condition { // === Payload Helpers === -// BuildPayload creates a Qdrant payload with separated internal and user fields +// BuildPayload creates a Qdrant payload with separated internal and user fields. +// Internal fields are stored at the top level, while user fields are stored under +// the "custom" prefix. If internal contains a "custom" key, it will be overwritten +// by the user fields map. func BuildPayload(internal map[string]any, user map[string]any) map[string]any { payload := make(map[string]any) @@ -323,6 +389,7 @@ func BuildPayload(internal map[string]any, user map[string]any) map[string]any { } // Add user fields under "custom" prefix + // Note: This will overwrite any "custom" key that was in the internal map if len(user) > 0 { payload[UserPayloadPrefix] = user } diff --git a/v1/qdrant/filters_test.go b/v1/qdrant/filters_test.go index b373b01..495b583 100644 --- a/v1/qdrant/filters_test.go +++ b/v1/qdrant/filters_test.go @@ -1,6 +1,7 @@ package qdrant import ( + "encoding/json" "testing" "time" ) @@ -263,6 +264,25 @@ func TestBuildConditions_NilConditionSet(t *testing.T) { } } +func TestBuildConditions_FiltersNilConditions(t *testing.T) { + // Test that buildConditions filters out nil conditions + cs := &ConditionSet{ + Conditions: []FilterCondition{ + TextCondition{Key: "city", Value: "London"}, + TimeRangeCondition{Key: "created_at", Value: TimeRange{}}, // Empty range returns nil + TextAnyCondition{Key: "status", Values: []string{}}, // Empty slice returns nil + BoolCondition{Key: "active", Value: true}, + }, + } + result := buildConditions(cs) + + // Should only have 2 conditions (TextCondition and BoolCondition) + // Empty TimeRange and empty TextAnyCondition should be filtered out + if len(result) != 2 { + t.Errorf("expected 2 conditions (nil ones filtered out), got %d", len(result)) + } +} + func TestTextCondition_ToQdrantCondition(t *testing.T) { c := TextCondition{Key: "city", Value: "London"} result := c.ToQdrantCondition() @@ -578,6 +598,47 @@ func TestBuildFilter_WithTextAnyCondition(t *testing.T) { } } +func TestTextAnyCondition_EmptySlice(t *testing.T) { + // Empty slice should return nil + c := TextAnyCondition{Key: "city", Values: []string{}} + result := c.ToQdrantCondition() + + if result != nil { + t.Errorf("expected nil for empty slice, got %v", result) + } +} + +func TestIntAnyCondition_EmptySlice(t *testing.T) { + // Empty slice should return nil + c := IntAnyCondition{Key: "priority", Values: []int64{}} + result := c.ToQdrantCondition() + + if result != nil { + t.Errorf("expected nil for empty slice, got %v", result) + } +} + +func TestBuildFilter_WithEmptyTextAnyCondition(t *testing.T) { + // Empty TextAnyCondition should be filtered out + filters := &FilterSet{ + Must: &ConditionSet{ + Conditions: []FilterCondition{ + TextAnyCondition{Key: "city", Values: []string{}}, + TextCondition{Key: "status", Value: "active"}, + }, + }, + } + result := buildFilter(filters) + + if result == nil { + t.Fatal("expected filter, got nil") + } + // Should only have the TextCondition, empty TextAnyCondition should be filtered out + if len(result.Must) != 1 { + t.Errorf("expected 1 Must condition (empty one filtered out), got %d", len(result.Must)) + } +} + // === MatchExceptCondition Tests === func TestTextExceptCondition_ToQdrantCondition(t *testing.T) { @@ -626,6 +687,47 @@ func TestBuildFilter_WithTextExceptCondition(t *testing.T) { } } +func TestTextExceptCondition_EmptySlice(t *testing.T) { + // Empty slice should return nil + c := TextExceptCondition{Key: "city", Values: []string{}} + result := c.ToQdrantCondition() + + if result != nil { + t.Errorf("expected nil for empty slice, got %v", result) + } +} + +func TestIntExceptCondition_EmptySlice(t *testing.T) { + // Empty slice should return nil + c := IntExceptCondition{Key: "priority", Values: []int64{}} + result := c.ToQdrantCondition() + + if result != nil { + t.Errorf("expected nil for empty slice, got %v", result) + } +} + +func TestBuildFilter_WithEmptyTextExceptCondition(t *testing.T) { + // Empty TextExceptCondition should be filtered out + filters := &FilterSet{ + MustNot: &ConditionSet{ + Conditions: []FilterCondition{ + TextExceptCondition{Key: "city", Values: []string{}}, + BoolCondition{Key: "deleted", Value: true}, + }, + }, + } + result := buildFilter(filters) + + if result == nil { + t.Fatal("expected filter, got nil") + } + // Should only have the BoolCondition, empty TextExceptCondition should be filtered out + if len(result.MustNot) != 1 { + t.Errorf("expected 1 MustNot condition (empty one filtered out), got %d", len(result.MustNot)) + } +} + // === NumericRangeCondition Tests === func TestNumericRangeCondition_ToQdrantCondition(t *testing.T) { @@ -856,6 +958,128 @@ func TestNumericRangeCondition_MarshalJSON(t *testing.T) { } } +func TestTimeRangeCondition_UnmarshalJSON(t *testing.T) { + jsonData := `{ + "field": "created_at", + "atOrAfter": "2024-01-01T00:00:00Z", + "before": "2024-12-31T00:00:00Z" + }` + + var c TimeRangeCondition + err := json.Unmarshal([]byte(jsonData), &c) + if err != nil { + t.Fatalf("UnmarshalJSON failed: %v", err) + } + + if c.Key != "created_at" { + t.Errorf("expected field 'created_at', got %q", c.Key) + } + if c.Value.Gte == nil { + t.Error("expected Gte to be set") + } + if c.Value.Lt == nil { + t.Error("expected Lt to be set") + } + if c.Value.Gte != nil { + expected := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + if !c.Value.Gte.Equal(expected) { + t.Errorf("expected Gte %v, got %v", expected, c.Value.Gte) + } + } +} + +func TestTimeRangeCondition_RoundTripJSON(t *testing.T) { + startTime := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + endTime := time.Date(2024, 12, 31, 0, 0, 0, 0, time.UTC) + + original := TimeRangeCondition{ + Key: "created_at", + Value: TimeRange{ + Gte: &startTime, + Lt: &endTime, + }, + } + + data, err := json.Marshal(original) + if err != nil { + t.Fatalf("Marshal failed: %v", err) + } + + var unmarshaled TimeRangeCondition + err = json.Unmarshal(data, &unmarshaled) + if err != nil { + t.Fatalf("Unmarshal failed: %v", err) + } + + if unmarshaled.Key != original.Key { + t.Errorf("field mismatch: expected %q, got %q", original.Key, unmarshaled.Key) + } + if unmarshaled.Value.Gte == nil || !unmarshaled.Value.Gte.Equal(*original.Value.Gte) { + t.Errorf("Gte mismatch: expected %v, got %v", original.Value.Gte, unmarshaled.Value.Gte) + } + if unmarshaled.Value.Lt == nil || !unmarshaled.Value.Lt.Equal(*original.Value.Lt) { + t.Errorf("Lt mismatch: expected %v, got %v", original.Value.Lt, unmarshaled.Value.Lt) + } +} + +func TestNumericRangeCondition_UnmarshalJSON(t *testing.T) { + jsonData := `{ + "field": "price", + "greaterThanOrEqualTo": 100.0, + "lessThanOrEqualTo": 500.0 + }` + + var c NumericRangeCondition + err := json.Unmarshal([]byte(jsonData), &c) + if err != nil { + t.Fatalf("UnmarshalJSON failed: %v", err) + } + + if c.Key != "price" { + t.Errorf("expected field 'price', got %q", c.Key) + } + if c.Value.Gte == nil || *c.Value.Gte != 100.0 { + t.Errorf("expected Gte to be 100.0, got %v", c.Value.Gte) + } + if c.Value.Lte == nil || *c.Value.Lte != 500.0 { + t.Errorf("expected Lte to be 500.0, got %v", c.Value.Lte) + } +} + +func TestNumericRangeCondition_RoundTripJSON(t *testing.T) { + minPrice := 100.0 + maxPrice := 500.0 + + original := NumericRangeCondition{ + Key: "price", + Value: NumericRange{ + Gte: &minPrice, + Lte: &maxPrice, + }, + } + + data, err := json.Marshal(original) + if err != nil { + t.Fatalf("Marshal failed: %v", err) + } + + var unmarshaled NumericRangeCondition + err = json.Unmarshal(data, &unmarshaled) + if err != nil { + t.Fatalf("Unmarshal failed: %v", err) + } + + if unmarshaled.Key != original.Key { + t.Errorf("field mismatch: expected %q, got %q", original.Key, unmarshaled.Key) + } + if unmarshaled.Value.Gte == nil || *unmarshaled.Value.Gte != *original.Value.Gte { + t.Errorf("Gte mismatch: expected %v, got %v", original.Value.Gte, unmarshaled.Value.Gte) + } + if unmarshaled.Value.Lte == nil || *unmarshaled.Value.Lte != *original.Value.Lte { + t.Errorf("Lte mismatch: expected %v, got %v", original.Value.Lte, unmarshaled.Value.Lte) + } +} + // === Complex Filter Tests === func TestBuildFilter_ComplexCombination(t *testing.T) { From df523df081d2c8be24cc63de5f63b4e4f7fa0276 Mon Sep 17 00:00:00 2001 From: Shinu Joseph Date: Tue, 9 Dec 2025 12:44:47 +0100 Subject: [PATCH 5/5] chore: fix JSON tags to match Qdrant --- v1/qdrant/filters.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/v1/qdrant/filters.go b/v1/qdrant/filters.go index a4a3d9b..20222e8 100644 --- a/v1/qdrant/filters.go +++ b/v1/qdrant/filters.go @@ -258,9 +258,9 @@ type ConditionSet struct { // }, // } type FilterSet struct { - Must *ConditionSet `json:"with,omitempty"` // AND - all conditions must match - Should *ConditionSet `json:"withOneOf,omitempty"` // OR - at least one condition must match - MustNot *ConditionSet `json:"without,omitempty"` // NOT - none of the conditions should match + Must *ConditionSet `json:"must,omitempty"` // AND - all conditions must match + Should *ConditionSet `json:"should,omitempty"` // OR - at least one condition must match + MustNot *ConditionSet `json:"mustNot,omitempty"` // NOT - none of the conditions should match } // buildFilter constructs a Qdrant filter from FilterSet