HYPERFLEET-536 - feat: add condition subfield queries for selective Sentinel polling

docs/search.md

@@ -44,6 +44,7 @@ The HyperFleet API uses the [Tree Search Language (TSL)](https://github.com/yaac
 | `updated_by` | string | Last updater email | `updated_by='user@example.com'` |
 | `labels.<key>` | string | Label value | `labels.environment='production'` |
 | `status.conditions.<Type>` | string | Condition status | `status.conditions.Ready='True'` |
+| `status.conditions.<Type>.<Subfield>` | varies | Condition subfield | `status.conditions.Ready.last_updated_time < '...'` |
 
 ```bash
 # Find cluster by name
@@ -99,7 +100,7 @@ Query resources by status conditions: `status.conditions.<Type>='<Status>'`
 
 Condition types must be PascalCase (`Ready`, `Available`) and status must be `True` or `False` for resource conditions.
 
-**Note:** Only the `=` operator is supported for condition queries. Other  operators (`!=`, `<`, `>`, `in`, etc.) will return an error. Use `not` with  conditions cautiously — `not status.conditions.Ready='True'` does not work as expected.
+**Note:** Only the `=` operator is supported for condition queries. Other operators (`!=`, `<`, `>`, `in`, etc.) will return an error. The `NOT` operator is not supported with condition queries (`status.conditions.<Type>` or `status.conditions.<Type>.<Subfield>`) and will return a `400 Bad Request` error. Use the inverse condition value instead (e.g., `status.conditions.Ready='False'` rather than `NOT status.conditions.Ready='True'`).
 
 ```bash
 # Find available clusters
@@ -111,6 +112,45 @@ curl -G "http://localhost:8000/api/hyperfleet/v1/clusters" \
   --data-urlencode "search=status.conditions.Ready='False'"
 ```
 
+## Condition Subfield Queries
+
+Query resources by condition subfields such as timestamps and observed generation:
+
+```text
+status.conditions.<Type>.<Subfield> <op> '<Value>'
+```
+
+### Supported Subfields
+
+| Subfield | Type | Description |
+|----------|------|-------------|
+| `last_updated_time` | TIMESTAMPTZ | When the condition was last updated |
+| `last_transition_time` | TIMESTAMPTZ | When the condition status last changed |
+| `observed_generation` | INTEGER | Last generation processed by the condition |
+
+### Supported Operators
+
+Condition subfields support comparison operators: `=`, `!=`, `<`, `<=`, `>`, `>=`.
+
+> **Note**: `status.conditions.<Type>` (without subfield) only supports the `=` operator.
+> The `NOT` operator is not supported with any condition expression — neither `status.conditions.<Type>` nor `status.conditions.<Type>.<Subfield>` (e.g., `status.conditions.Ready.last_updated_time`). Using `NOT` with these expressions returns a `400 Bad Request` error. Restructure queries using `AND`/`OR` or the inverse condition value instead.
+
+```bash
+# Find clusters where Ready condition hasn't been updated in the last hour
+curl -G "http://localhost:8000/api/hyperfleet/v1/clusters" \
+  --data-urlencode "search=status.conditions.Ready.last_updated_time < '2026-03-06T14:00:00Z'"
+
+# Find stale-ready resources (Sentinel selective polling use case)
+curl -G "http://localhost:8000/api/hyperfleet/v1/clusters" \
+  --data-urlencode "search=status.conditions.Ready='True' AND status.conditions.Ready.last_updated_time < '2026-03-06T14:00:00Z'"
+
+# Find clusters with observed_generation below a threshold (uses unquoted integer)
+curl -G "http://localhost:8000/api/hyperfleet/v1/clusters" \
+  --data-urlencode "search=status.conditions.Ready.observed_generation < 5"
+```
+
+Time subfields require RFC3339 format (e.g., `2026-01-01T00:00:00Z`). Integer subfields use unquoted numeric values.
+
 ## Complex Queries
 
 Combine multiple conditions using `and`, `or`, `not`, and parentheses `()`:

pkg/dao/cluster.go

@@ -3,6 +3,7 @@ package dao
 import (
 	"bytes"
 	"context"
+	"time"
 
 	"gorm.io/gorm/clause"
 
@@ -57,9 +58,15 @@ func (d *sqlClusterDao) Replace(ctx context.Context, cluster *api.Cluster) (*api
 		return nil, err
 	}
 
-	// Compare spec: if changed, increment generation
+	// Compare spec: if changed, increment generation and reset Ready=False
 	if !bytes.Equal(existing.Spec, cluster.Spec) {
 		cluster.Generation = existing.Generation + 1
+		updatedConditions, err := resetReadyConditionOnSpecChange(existing.StatusConditions, time.Now())
+		if err != nil {
+			db.MarkForRollback(ctx, err)
+			return nil, err
+		}
+		cluster.StatusConditions = updatedConditions
 	} else {
 		// Spec unchanged, preserve generation
 		cluster.Generation = existing.Generation

pkg/dao/conditions.go

@@ -0,0 +1,46 @@
+package dao
+
+import (
+	"encoding/json"
+	"time"
+
+	"gorm.io/datatypes"
+
+	"github.com/openshift-hyperfleet/hyperfleet-api/pkg/api"
+)
+
+// resetReadyConditionOnSpecChange flips Ready=False when the spec changes (generation incremented).
+// This ensures Sentinel's selective querying immediately picks up the resource
+// via the "not-ready resources" query on the next poll cycle.
+// Available is intentionally left unchanged — it reflects last known good state at any generation.
+func resetReadyConditionOnSpecChange(existingConditions datatypes.JSON, now time.Time) (datatypes.JSON, error) {
+	if len(existingConditions) == 0 {
+		return existingConditions, nil
+	}
+
+	var conditions []api.ResourceCondition
+	if err := json.Unmarshal(existingConditions, &conditions); err != nil {
+		return existingConditions, nil
+	}
+
+	changed := false
+	for i := range conditions {
+		if conditions[i].Type == api.ConditionTypeReady && conditions[i].Status == api.ConditionTrue {
+			conditions[i].Status = api.ConditionFalse
+			conditions[i].LastTransitionTime = now
+			conditions[i].LastUpdatedTime = now
+			reason := "SpecChanged"
+			message := "Spec updated, awaiting adapters to reconcile at new generation"
+			conditions[i].Reason = &reason
+			conditions[i].Message = &message
+			changed = true
+			break
+		}
+	}
+
+	if !changed {
+		return existingConditions, nil
+	}
+
+	return json.Marshal(conditions)
+}

pkg/dao/conditions_test.go

@@ -0,0 +1,98 @@
+package dao
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	. "github.com/onsi/gomega"
+	"github.com/openshift-hyperfleet/hyperfleet-api/pkg/api"
+)
+
+func TestResetReadyConditionOnSpecChange(t *testing.T) {
+	now := time.Now()
+
+	tests := []struct {
+		name               string
+		conditions         []api.ResourceCondition
+		expectReadyFalse   bool
+		expectReasonChange bool
+	}{
+		{
+			name: "Ready=True is flipped to False",
+			conditions: []api.ResourceCondition{
+				{Type: "Available", Status: api.ConditionTrue},
+				{Type: "Ready", Status: api.ConditionTrue},
+			},
+			expectReadyFalse:   true,
+			expectReasonChange: true,
+		},
+		{
+			name: "Ready=False stays False",
+			conditions: []api.ResourceCondition{
+				{Type: "Available", Status: api.ConditionTrue},
+				{Type: "Ready", Status: api.ConditionFalse},
+			},
+			expectReadyFalse:   true,
+			expectReasonChange: false,
+		},
+		{
+			name: "Available is not changed",
+			conditions: []api.ResourceCondition{
+				{Type: "Available", Status: api.ConditionTrue},
+				{Type: "Ready", Status: api.ConditionTrue},
+			},
+			expectReadyFalse: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			RegisterTestingT(t)
+
+			input, err := json.Marshal(tt.conditions)
+			Expect(err).ToNot(HaveOccurred())
+
+			result, err := resetReadyConditionOnSpecChange(input, now)
+			Expect(err).ToNot(HaveOccurred())
+
+			var resultConditions []api.ResourceCondition
+			Expect(json.Unmarshal(result, &resultConditions)).To(Succeed())
+
+			for _, cond := range resultConditions {
+				switch cond.Type {
+				case "Ready":
+					if tt.expectReadyFalse {
+						Expect(cond.Status).To(Equal(api.ConditionFalse))
+					}
+					if tt.expectReasonChange {
+						Expect(cond.Reason).ToNot(BeNil())
+						Expect(*cond.Reason).To(Equal("SpecChanged"))
+						Expect(cond.LastTransitionTime.Equal(now)).To(BeTrue())
+					}
+				case "Available":
+					Expect(cond.Status).To(Equal(api.ConditionTrue))
+				}
+			}
+		})
+	}
+}
+
+func TestResetReadyConditionOnSpecChange_EmptyConditions(t *testing.T) {
+	RegisterTestingT(t)
+	now := time.Now()
+
+	result, err := resetReadyConditionOnSpecChange(nil, now)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(result).To(BeNil())
+}
+
+func TestResetReadyConditionOnSpecChange_InvalidJSON(t *testing.T) {
+	RegisterTestingT(t)
+	now := time.Now()
+
+	input := []byte(`not valid json`)
+	result, err := resetReadyConditionOnSpecChange(input, now)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(string(result)).To(Equal(string(input)))
+}

pkg/dao/node_pool.go

@@ -3,6 +3,7 @@ package dao
 import (
 	"bytes"
 	"context"
+	"time"
 
 	"gorm.io/gorm/clause"
 
@@ -57,9 +58,15 @@ func (d *sqlNodePoolDao) Replace(ctx context.Context, nodePool *api.NodePool) (*
 		return nil, err
 	}
 
-	// Compare spec: if changed, increment generation
+	// Compare spec: if changed, increment generation and reset Ready=False
 	if !bytes.Equal(existing.Spec, nodePool.Spec) {
 		nodePool.Generation = existing.Generation + 1
+		updatedConditions, err := resetReadyConditionOnSpecChange(existing.StatusConditions, time.Now())
+		if err != nil {
+			db.MarkForRollback(ctx, err)
+			return nil, err
+		}
+		nodePool.StatusConditions = updatedConditions
 	} else {
 		// Spec unchanged, preserve generation
 		nodePool.Generation = existing.Generation

pkg/db/migrations/202603100001_add_conditions_subfield_indexes.go

@@ -0,0 +1,76 @@
+package migrations
+
+import (
+	"github.com/go-gormigrate/gormigrate/v2"
+	"gorm.io/gorm"
+)
+
+// addConditionsSubfieldIndexes adds expression indexes for condition subfield queries.
+// The primary use case is Sentinel polling for stale-ready resources:
+//
+//	status.conditions.Ready.last_updated_time < '...'
+//
+// PostgreSQL requires all functions in expression indexes to be IMMUTABLE.
+// The built-in CAST(text AS TIMESTAMPTZ) is STABLE (depends on timezone settings),
+// so we create an IMMUTABLE wrapper. This is safe because our timestamps are always
+// stored in RFC3339 format with explicit timezone offsets.
+//
+// LIMITATION: The query builder uses Squirrel, which treats '?' in jsonpath filter
+// syntax ($[*] ? (...)) as bind placeholders, forcing parameterized queries that
+// PostgreSQL's planner cannot match against these expression indexes. HYPERFLEET-736
+// evaluates generated columns or table normalization as a proper fix.
+func addConditionsSubfieldIndexes() *gormigrate.Migration {
+	return &gormigrate.Migration{
+		ID: "202603100001",
+		Migrate: func(tx *gorm.DB) error {
+			// Create IMMUTABLE wrapper for text-to-timestamptz conversion.
+			// Required because PostgreSQL's built-in cast is STABLE, not IMMUTABLE.
+			if err := tx.Exec(`
+				CREATE OR REPLACE FUNCTION immutable_timestamptz(text)
+				RETURNS TIMESTAMPTZ
+				LANGUAGE SQL IMMUTABLE STRICT
+				AS $$ SELECT $1::timestamptz $$;
+			`).Error; err != nil {
+				return err
+			}
+
+			// Expression index on clusters for Ready condition last_updated_time
+			if err := tx.Exec(`
+				CREATE INDEX IF NOT EXISTS idx_clusters_ready_last_updated_time
+				ON clusters USING BTREE ((
+					immutable_timestamptz(
+						jsonb_path_query_first(status_conditions, '$[*] ? (@.type == "Ready")') ->> 'last_updated_time'
+					)
+				));
+			`).Error; err != nil {
+				return err
+			}
+
+			// Expression index on node_pools for Ready condition last_updated_time
+			if err := tx.Exec(`
+				CREATE INDEX IF NOT EXISTS idx_node_pools_ready_last_updated_time
+				ON node_pools USING BTREE ((
+					immutable_timestamptz(
+						jsonb_path_query_first(status_conditions, '$[*] ? (@.type == "Ready")') ->> 'last_updated_time'
+					)
+				));
+			`).Error; err != nil {
+				return err
+			}
+
+			return nil
+		},
+		Rollback: func(tx *gorm.DB) error {
+			if err := tx.Exec("DROP INDEX IF EXISTS idx_clusters_ready_last_updated_time;").Error; err != nil {
+				return err
+			}
+			if err := tx.Exec("DROP INDEX IF EXISTS idx_node_pools_ready_last_updated_time;").Error; err != nil {
+				return err
+			}
+			if err := tx.Exec("DROP FUNCTION IF EXISTS immutable_timestamptz(text);").Error; err != nil {
+				return err
+			}
+			return nil
+		},
+	}
+}

pkg/db/migrations/migration_structs.go

@@ -32,6 +32,7 @@ var MigrationList = []*gormigrate.Migration{
 	addNodePools(),
 	addAdapterStatus(),
 	addConditionsGinIndex(),
+	addConditionsSubfieldIndexes(),
 }
 
 // Model represents the base model struct. All entities will have this struct embedded.