From 0ba753e39b7609d2df56b755d9b938ec106ecff3 Mon Sep 17 00:00:00 2001 From: Ben Jee Date: Thu, 2 Oct 2025 16:59:00 -0700 Subject: [PATCH 1/4] Add telemetry collection for referenced inference pools --- internal/controller/telemetry/collector.go | 4 ++++ internal/controller/telemetry/collector_test.go | 10 ++++++++++ internal/controller/telemetry/data.avdl | 3 +++ internal/controller/telemetry/data_test.go | 4 +++- .../ngfresourcecounts_attributes_generated.go | 1 + tests/suite/telemetry_test.go | 1 + 6 files changed, 22 insertions(+), 1 deletion(-) diff --git a/internal/controller/telemetry/collector.go b/internal/controller/telemetry/collector.go index e06da3f0b8..0f098bf48e 100644 --- a/internal/controller/telemetry/collector.go +++ b/internal/controller/telemetry/collector.go @@ -105,6 +105,8 @@ type NGFResourceCounts struct { UpstreamSettingsPolicyCount int64 // GatewayAttachedNpCount is the total number of NginxProxy resources that are attached to a Gateway. GatewayAttachedNpCount int64 + // InferencePoolCount is the number of InferencePools that are referenced by at least one Route. + InferencePoolCount int64 } // DataCollectorConfig holds configuration parameters for DataCollectorImpl. @@ -265,6 +267,8 @@ func collectGraphResourceCount( ngfResourceCounts.GatewayAttachedNpCount = gatewayAttachedNPCount + ngfResourceCounts.InferencePoolCount = int64(len(g.ReferencedInferencePools)) + return ngfResourceCounts } diff --git a/internal/controller/telemetry/collector_test.go b/internal/controller/telemetry/collector_test.go index 8c749fdfbe..e25bccc590 100644 --- a/internal/controller/telemetry/collector_test.go +++ b/internal/controller/telemetry/collector_test.go @@ -417,6 +417,11 @@ var _ = Describe("Collector", Ordered, func() { }, }, }, + ReferencedInferencePools: map[types.NamespacedName]*graph.ReferencedInferencePool{ + {Namespace: "test", Name: "inferencePool-1"}: {}, + {Namespace: "test", Name: "inferencePool-2"}: {}, + {Namespace: "test", Name: "inferencePool-3"}: {}, + }, } configs := []*dataplane.Configuration{ @@ -487,6 +492,7 @@ var _ = Describe("Collector", Ordered, func() { SnippetsFilterCount: 3, UpstreamSettingsPolicyCount: 1, GatewayAttachedNpCount: 2, + InferencePoolCount: 3, } expData.ClusterVersion = "1.29.2" expData.ClusterPlatform = "kind" @@ -700,6 +706,9 @@ var _ = Describe("Collector", Ordered, func() { BackendTLSPolicies: map[types.NamespacedName]*graph.BackendTLSPolicy{ {Namespace: "test", Name: "BackendTLSPolicy-1"}: {}, }, + ReferencedInferencePools: map[types.NamespacedName]*graph.ReferencedInferencePool{ + {Namespace: "test", Name: "inferencePool-1"}: {}, + }, } config1 = []*dataplane.Configuration{ @@ -781,6 +790,7 @@ var _ = Describe("Collector", Ordered, func() { UpstreamSettingsPolicyCount: 1, GatewayAttachedNpCount: 1, BackendTLSPolicyCount: 1, + InferencePoolCount: 1, } expData.NginxPodCount = 1 diff --git a/internal/controller/telemetry/data.avdl b/internal/controller/telemetry/data.avdl index c19881315a..d3b0aa8a9d 100644 --- a/internal/controller/telemetry/data.avdl +++ b/internal/controller/telemetry/data.avdl @@ -105,6 +105,9 @@ attached at the Gateway level. */ /** GatewayAttachedNpCount is the total number of NginxProxy resources that are attached to a Gateway. */ long? GatewayAttachedNpCount = null; + /** InferencePoolCount is the number of InferencePools that are referenced by at least one Route. */ + long? InferencePoolCount = null; + /** NginxPodCount is the total number of Nginx data plane Pods. */ long? NginxPodCount = null; diff --git a/internal/controller/telemetry/data_test.go b/internal/controller/telemetry/data_test.go index 49c8e3543c..e817a0f494 100644 --- a/internal/controller/telemetry/data_test.go +++ b/internal/controller/telemetry/data_test.go @@ -41,6 +41,7 @@ func TestDataAttributes(t *testing.T) { SnippetsFilterCount: 13, UpstreamSettingsPolicyCount: 14, GatewayAttachedNpCount: 15, + InferencePoolCount: 16, }, SnippetsFiltersDirectives: []string{"main-three-count", "http-two-count", "server-one-count"}, SnippetsFiltersDirectivesCount: []int64{3, 2, 1}, @@ -83,6 +84,7 @@ func TestDataAttributes(t *testing.T) { attribute.Int64("SnippetsFilterCount", 13), attribute.Int64("UpstreamSettingsPolicyCount", 14), attribute.Int64("GatewayAttachedNpCount", 15), + attribute.Int64("InferencePoolCount", 16), attribute.Int64("NginxPodCount", 3), attribute.Int64("ControlPlanePodCount", 3), attribute.Bool("NginxOneConnectionEnabled", true), @@ -129,6 +131,7 @@ func TestDataAttributesWithEmptyData(t *testing.T) { attribute.Int64("SnippetsFilterCount", 0), attribute.Int64("UpstreamSettingsPolicyCount", 0), attribute.Int64("GatewayAttachedNpCount", 0), + attribute.Int64("InferencePoolCount", 0), attribute.Int64("NginxPodCount", 0), attribute.Int64("ControlPlanePodCount", 0), attribute.Bool("NginxOneConnectionEnabled", false), @@ -137,6 +140,5 @@ func TestDataAttributesWithEmptyData(t *testing.T) { result := data.Attributes() g := NewWithT(t) - g.Expect(result).To(Equal(expected)) } diff --git a/internal/controller/telemetry/ngfresourcecounts_attributes_generated.go b/internal/controller/telemetry/ngfresourcecounts_attributes_generated.go index 3073f15eb4..a755083c1b 100644 --- a/internal/controller/telemetry/ngfresourcecounts_attributes_generated.go +++ b/internal/controller/telemetry/ngfresourcecounts_attributes_generated.go @@ -28,6 +28,7 @@ func (d *NGFResourceCounts) Attributes() []attribute.KeyValue { attrs = append(attrs, attribute.Int64("SnippetsFilterCount", d.SnippetsFilterCount)) attrs = append(attrs, attribute.Int64("UpstreamSettingsPolicyCount", d.UpstreamSettingsPolicyCount)) attrs = append(attrs, attribute.Int64("GatewayAttachedNpCount", d.GatewayAttachedNpCount)) + attrs = append(attrs, attribute.Int64("InferencePoolCount", d.InferencePoolCount)) return attrs } diff --git a/tests/suite/telemetry_test.go b/tests/suite/telemetry_test.go index 2ad0c0b3a0..7ddce2aa26 100644 --- a/tests/suite/telemetry_test.go +++ b/tests/suite/telemetry_test.go @@ -96,6 +96,7 @@ var _ = Describe("Telemetry test with OTel collector", Label("telemetry"), func( "NginxPodCount: Int(0)", "ControlPlanePodCount: Int(1)", "NginxOneConnectionEnabled: Bool(false)", + "InferencePoolCount: Int(0)", }, ) }) From 61c5a617a195f4d440e7da769299f692237fcd1c Mon Sep 17 00:00:00 2001 From: Ben Jee Date: Fri, 3 Oct 2025 12:15:26 -0700 Subject: [PATCH 2/4] Add back space --- internal/controller/telemetry/data_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/controller/telemetry/data_test.go b/internal/controller/telemetry/data_test.go index e817a0f494..51030dbabe 100644 --- a/internal/controller/telemetry/data_test.go +++ b/internal/controller/telemetry/data_test.go @@ -140,5 +140,6 @@ func TestDataAttributesWithEmptyData(t *testing.T) { result := data.Attributes() g := NewWithT(t) + g.Expect(result).To(Equal(expected)) } From 933d3bf2e49f9344ee83dfff2a57a03689fe890f Mon Sep 17 00:00:00 2001 From: Ben Jee Date: Fri, 3 Oct 2025 12:56:25 -0700 Subject: [PATCH 3/4] Fix telemetry test --- examples/inference-pool/gateway.yaml | 10 ++++++++++ examples/inference-pool/httproute.yaml | 26 ++++++++++++++++++++++++++ tests/suite/telemetry_test.go | 2 +- 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 examples/inference-pool/gateway.yaml create mode 100644 examples/inference-pool/httproute.yaml diff --git a/examples/inference-pool/gateway.yaml b/examples/inference-pool/gateway.yaml new file mode 100644 index 0000000000..7c3a4b30e4 --- /dev/null +++ b/examples/inference-pool/gateway.yaml @@ -0,0 +1,10 @@ +kind: Gateway +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: inference-gateway +spec: + gatewayClassName: nginx + listeners: + - name: http + port: 80 + protocol: HTTP diff --git a/examples/inference-pool/httproute.yaml b/examples/inference-pool/httproute.yaml new file mode 100644 index 0000000000..ffb2fefcc0 --- /dev/null +++ b/examples/inference-pool/httproute.yaml @@ -0,0 +1,26 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: llm-route +spec: + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: inference-gateway + rules: + - backendRefs: + - group: inference.networking.k8s.io + kind: InferencePool + name: vllm-llama3-8b-instruct + matches: + - path: + type: PathPrefix + value: / + - backendRefs: + - group: inference.networking.k8s.io + kind: InferencePool + name: vllm-llama3-8b-instructerr + matches: + - path: + type: PathPrefix + value: / diff --git a/tests/suite/telemetry_test.go b/tests/suite/telemetry_test.go index 7ddce2aa26..e8837747cb 100644 --- a/tests/suite/telemetry_test.go +++ b/tests/suite/telemetry_test.go @@ -93,10 +93,10 @@ var _ = Describe("Telemetry test with OTel collector", Label("telemetry"), func( "SnippetsFilterCount: Int(0)", "UpstreamSettingsPolicyCount: Int(0)", "GatewayAttachedNpCount: Int(0)", + "InferencePoolCount: Int(0)", "NginxPodCount: Int(0)", "ControlPlanePodCount: Int(1)", "NginxOneConnectionEnabled: Bool(false)", - "InferencePoolCount: Int(0)", }, ) }) From 9cedf155eaa3b80ff6afcc802e843ee43a6b7bd3 Mon Sep 17 00:00:00 2001 From: Ben Jee Date: Fri, 3 Oct 2025 12:57:19 -0700 Subject: [PATCH 4/4] Remove added files --- examples/inference-pool/gateway.yaml | 10 ---------- examples/inference-pool/httproute.yaml | 26 -------------------------- 2 files changed, 36 deletions(-) delete mode 100644 examples/inference-pool/gateway.yaml delete mode 100644 examples/inference-pool/httproute.yaml diff --git a/examples/inference-pool/gateway.yaml b/examples/inference-pool/gateway.yaml deleted file mode 100644 index 7c3a4b30e4..0000000000 --- a/examples/inference-pool/gateway.yaml +++ /dev/null @@ -1,10 +0,0 @@ -kind: Gateway -apiVersion: gateway.networking.k8s.io/v1 -metadata: - name: inference-gateway -spec: - gatewayClassName: nginx - listeners: - - name: http - port: 80 - protocol: HTTP diff --git a/examples/inference-pool/httproute.yaml b/examples/inference-pool/httproute.yaml deleted file mode 100644 index ffb2fefcc0..0000000000 --- a/examples/inference-pool/httproute.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: gateway.networking.k8s.io/v1 -kind: HTTPRoute -metadata: - name: llm-route -spec: - parentRefs: - - group: gateway.networking.k8s.io - kind: Gateway - name: inference-gateway - rules: - - backendRefs: - - group: inference.networking.k8s.io - kind: InferencePool - name: vllm-llama3-8b-instruct - matches: - - path: - type: PathPrefix - value: / - - backendRefs: - - group: inference.networking.k8s.io - kind: InferencePool - name: vllm-llama3-8b-instructerr - matches: - - path: - type: PathPrefix - value: /