From 10a8fbab3f318e45c99262c05769bd1a5672c641 Mon Sep 17 00:00:00 2001 From: POOJA REDDY NATHALA Date: Tue, 1 Aug 2023 16:06:06 -0400 Subject: [PATCH] Adding pod_status_unknown as a Pod condition metric and removed the pod_status_initialized metric (#66) * Adding pod_status_unknown as a Pod condition metric and removed the pod_status_initialized metric --- internal/aws/containerinsight/const.go | 4 +- .../awscontainerinsightreceiver/README.md | 81 ++++----- .../internal/stores/podstore.go | 27 +-- .../internal/stores/podstore_test.go | 44 ++--- ...pod_Ready_Scheduled_Condition_Unknown.json | 156 ++++++++++++++++++ 5 files changed, 235 insertions(+), 77 deletions(-) create mode 100644 receiver/awscontainerinsightreceiver/internal/stores/test_resources/pod_Ready_Scheduled_Condition_Unknown.json diff --git a/internal/aws/containerinsight/const.go b/internal/aws/containerinsight/const.go index cd288506c9a6e..3ecde9ecbf952 100644 --- a/internal/aws/containerinsight/const.go +++ b/internal/aws/containerinsight/const.go @@ -114,7 +114,6 @@ const ( StatusUnknown = "status_unknown" StatusReady = "status_ready" StatusScheduled = "status_scheduled" - StatusInitialized = "status_initialized" RunningPodCount = "number_of_running_pods" RunningContainerCount = "number_of_running_containers" @@ -258,6 +257,9 @@ func init() { StatusTerminated: UnitCount, StatusWaiting: UnitCount, StatusWaitingReasonCrashed: UnitCount, + StatusUnknown: UnitCount, + StatusReady: UnitCount, + StatusScheduled: UnitCount, // cluster metrics NodeCount: UnitCount, diff --git a/receiver/awscontainerinsightreceiver/README.md b/receiver/awscontainerinsightreceiver/README.md index bbac91b70e898..1fe0e3b4d3a96 100644 --- a/receiver/awscontainerinsightreceiver/README.md +++ b/receiver/awscontainerinsightreceiver/README.md @@ -633,45 +633,48 @@ kubectl apply -f config.yaml

### Pod -| Metric | Unit | -|---------------------------------------|---------------| -| pod_cpu_limit | Millicore | -| pod_cpu_request | Millicore | -| pod_cpu_reserved_capacity | Percent | -| pod_cpu_usage_system | Millicore | -| pod_cpu_usage_total | Millicore | -| pod_cpu_usage_user | Millicore | -| pod_cpu_utilization | Percent | -| pod_cpu_utilization_over_pod_limit | Percent | -| pod_memory_cache | Bytes | -| pod_memory_failcnt | Count | -| pod_memory_hierarchical_pgfault | Count/Second | -| pod_memory_hierarchical_pgmajfault | Count/Second | -| pod_memory_limit | Bytes | -| pod_memory_mapped_file | Bytes | -| pod_memory_max_usage | Bytes | -| pod_memory_pgfault | Count/Second | -| pod_memory_pgmajfault | Count/Second | -| pod_memory_request | Bytes | -| pod_memory_reserved_capacity | Percent | -| pod_memory_rss | Bytes | -| pod_memory_swap | Bytes | -| pod_memory_usage | Bytes | -| pod_memory_utilization | Percent | -| pod_memory_utilization_over_pod_limit | Percent | -| pod_memory_working_set | Bytes | -| pod_network_rx_bytes | Bytes/Second | -| pod_network_rx_dropped | Count/Second | -| pod_network_rx_errors | Count/Second | -| pod_network_rx_packets | Count/Second | -| pod_network_total_bytes | Bytes/Second | -| pod_network_tx_bytes | Bytes/Second | -| pod_network_tx_dropped | Count/Second | -| pod_network_tx_errors | Count/Second | -| pod_network_tx_packets | Count/Second | -| pod_number_of_container_restarts | Count | -| pod_number_of_containers | Count | -| pod_number_of_running_containers | Count | +| Metric | Unit | +|---------------------------------------|--------------| +| pod_cpu_limit | Millicore | +| pod_cpu_request | Millicore | +| pod_cpu_reserved_capacity | Percent | +| pod_cpu_usage_system | Millicore | +| pod_cpu_usage_total | Millicore | +| pod_cpu_usage_user | Millicore | +| pod_cpu_utilization | Percent | +| pod_cpu_utilization_over_pod_limit | Percent | +| pod_memory_cache | Bytes | +| pod_memory_failcnt | Count | +| pod_memory_hierarchical_pgfault | Count/Second | +| pod_memory_hierarchical_pgmajfault | Count/Second | +| pod_memory_limit | Bytes | +| pod_memory_mapped_file | Bytes | +| pod_memory_max_usage | Bytes | +| pod_memory_pgfault | Count/Second | +| pod_memory_pgmajfault | Count/Second | +| pod_memory_request | Bytes | +| pod_memory_reserved_capacity | Percent | +| pod_memory_rss | Bytes | +| pod_memory_swap | Bytes | +| pod_memory_usage | Bytes | +| pod_memory_utilization | Percent | +| pod_memory_utilization_over_pod_limit | Percent | +| pod_memory_working_set | Bytes | +| pod_network_rx_bytes | Bytes/Second | +| pod_network_rx_dropped | Count/Second | +| pod_network_rx_errors | Count/Second | +| pod_network_rx_packets | Count/Second | +| pod_network_total_bytes | Bytes/Second | +| pod_network_tx_bytes | Bytes/Second | +| pod_network_tx_dropped | Count/Second | +| pod_network_tx_errors | Count/Second | +| pod_network_tx_packets | Count/Second | +| pod_number_of_container_restarts | Count | +| pod_number_of_containers | Count | +| pod_number_of_running_containers | Count | +| pod_status_ready | Count | +| pod_status_scheduled | Count | +| pod_status_unknown | Count | | Resource Attribute | |----------------------| diff --git a/receiver/awscontainerinsightreceiver/internal/stores/podstore.go b/receiver/awscontainerinsightreceiver/internal/stores/podstore.go index b7a8395c7af0d..b142362ca9345 100644 --- a/receiver/awscontainerinsightreceiver/internal/stores/podstore.go +++ b/receiver/awscontainerinsightreceiver/internal/stores/podstore.go @@ -53,14 +53,14 @@ var ( corev1.PodRunning: ci.MetricName(ci.TypePod, ci.StatusRunning), corev1.PodSucceeded: ci.MetricName(ci.TypePod, ci.StatusSucceeded), corev1.PodFailed: ci.MetricName(ci.TypePod, ci.StatusFailed), - corev1.PodUnknown: ci.MetricName(ci.TypePod, ci.StatusUnknown), } PodConditionMetricNames = map[corev1.PodConditionType]string{ - corev1.PodReady: ci.MetricName(ci.TypePod, ci.StatusReady), - corev1.PodScheduled: ci.MetricName(ci.TypePod, ci.StatusScheduled), - corev1.PodInitialized: ci.MetricName(ci.TypePod, ci.StatusInitialized), + corev1.PodReady: ci.MetricName(ci.TypePod, ci.StatusReady), + corev1.PodScheduled: ci.MetricName(ci.TypePod, ci.StatusScheduled), } + + PodConditionUnknownMetric = ci.MetricName(ci.TypePod, ci.StatusUnknown) ) type cachedEntry struct { @@ -570,16 +570,21 @@ func (p *PodStore) addPodConditionMetrics(metric CIMetric, pod *corev1.Pod) { metric.AddField(metricName, 0) } + metric.AddField(PodConditionUnknownMetric, 0) + for _, condition := range pod.Status.Conditions { - if condition.Status != corev1.ConditionTrue { - continue - } - conditionKey := condition.Type - statusMetricName, conditionKeyValid := PodConditionMetricNames[conditionKey] - if conditionKeyValid { - metric.AddField(statusMetricName, 1) + switch condition.Status { + case corev1.ConditionTrue: + if statusMetricName, ok := PodConditionMetricNames[condition.Type]; ok { + metric.AddField(statusMetricName, 1) + } + case corev1.ConditionUnknown: + if _, ok := PodConditionMetricNames[condition.Type]; ok { + metric.AddField(PodConditionUnknownMetric, 1) + } } + } } diff --git a/receiver/awscontainerinsightreceiver/internal/stores/podstore_test.go b/receiver/awscontainerinsightreceiver/internal/stores/podstore_test.go index 9743774d13b51..cad355b82fcec 100644 --- a/receiver/awscontainerinsightreceiver/internal/stores/podstore_test.go +++ b/receiver/awscontainerinsightreceiver/internal/stores/podstore_test.go @@ -339,14 +339,13 @@ func TestPodStore_addContainerCount(t *testing.T) { } const ( - PodFailedMetricName = "pod_status_failed" - PodPendingMetricName = "pod_status_pending" - PodRunningMetricName = "pod_status_running" - PodSucceededMetricName = "pod_status_succeeded" - PodUnknownMetricName = "pod_status_unknown" - PodReadyMetricName = "pod_status_ready" - PodScheduledMetricName = "pod_status_scheduled" - PodInitializedMetricName = "pod_status_initialized" + PodFailedMetricName = "pod_status_failed" + PodPendingMetricName = "pod_status_pending" + PodRunningMetricName = "pod_status_running" + PodSucceededMetricName = "pod_status_succeeded" + PodUnknownMetricName = "pod_status_unknown" + PodReadyMetricName = "pod_status_ready" + PodScheduledMetricName = "pod_status_scheduled" ) func TestPodStore_enhanced_metrics_disabled(t *testing.T) { @@ -356,7 +355,6 @@ func TestPodStore_enhanced_metrics_disabled(t *testing.T) { assert.False(t, decoratedResultMetric.HasField(PodPendingMetricName)) assert.False(t, decoratedResultMetric.HasField(PodRunningMetricName)) assert.False(t, decoratedResultMetric.HasField(PodSucceededMetricName)) - assert.False(t, decoratedResultMetric.HasField(PodUnknownMetricName)) } func TestPodStore_addStatus_adds_pod_failed_metric(t *testing.T) { @@ -366,7 +364,6 @@ func TestPodStore_addStatus_adds_pod_failed_metric(t *testing.T) { assert.Equal(t, 0, decoratedResultMetric.GetField(PodPendingMetricName)) assert.Equal(t, 0, decoratedResultMetric.GetField(PodRunningMetricName)) assert.Equal(t, 0, decoratedResultMetric.GetField(PodSucceededMetricName)) - assert.Equal(t, 0, decoratedResultMetric.GetField(PodUnknownMetricName)) } func TestPodStore_addStatus_adds_pod_pending_metric(t *testing.T) { @@ -376,7 +373,6 @@ func TestPodStore_addStatus_adds_pod_pending_metric(t *testing.T) { assert.Equal(t, 1, decoratedResultMetric.GetField(PodPendingMetricName)) assert.Equal(t, 0, decoratedResultMetric.GetField(PodRunningMetricName)) assert.Equal(t, 0, decoratedResultMetric.GetField(PodSucceededMetricName)) - assert.Equal(t, 0, decoratedResultMetric.GetField(PodUnknownMetricName)) } func TestPodStore_addStatus_adds_pod_running_metric(t *testing.T) { @@ -386,7 +382,6 @@ func TestPodStore_addStatus_adds_pod_running_metric(t *testing.T) { assert.Equal(t, 0, decoratedResultMetric.GetField(PodPendingMetricName)) assert.Equal(t, 1, decoratedResultMetric.GetField(PodRunningMetricName)) assert.Equal(t, 0, decoratedResultMetric.GetField(PodSucceededMetricName)) - assert.Equal(t, 0, decoratedResultMetric.GetField(PodUnknownMetricName)) } func TestPodStore_addStatus_adds_pod_succeeded_metric(t *testing.T) { @@ -396,41 +391,38 @@ func TestPodStore_addStatus_adds_pod_succeeded_metric(t *testing.T) { assert.Equal(t, 0, decoratedResultMetric.GetField(PodPendingMetricName)) assert.Equal(t, 0, decoratedResultMetric.GetField(PodRunningMetricName)) assert.Equal(t, 1, decoratedResultMetric.GetField(PodSucceededMetricName)) - assert.Equal(t, 0, decoratedResultMetric.GetField(PodUnknownMetricName)) -} - -func TestPodStore_addStatus_adds_pod_unknown_metric(t *testing.T) { - decoratedResultMetric := runAddStatusToGetDecoratedCIMetric("./test_resources/pod_in_phase_unknown.json", true) - - assert.Equal(t, 0, decoratedResultMetric.GetField(PodFailedMetricName)) - assert.Equal(t, 0, decoratedResultMetric.GetField(PodPendingMetricName)) - assert.Equal(t, 0, decoratedResultMetric.GetField(PodRunningMetricName)) - assert.Equal(t, 0, decoratedResultMetric.GetField(PodSucceededMetricName)) - assert.Equal(t, 1, decoratedResultMetric.GetField(PodUnknownMetricName)) } func TestPodStore_addStatus_enhanced_metrics_disabled(t *testing.T) { decoratedResultMetric := runAddStatusToGetDecoratedCIMetric("./test_resources/all_pod_conditions_valid.json", false) - assert.False(t, decoratedResultMetric.HasField(PodInitializedMetricName)) assert.False(t, decoratedResultMetric.HasField(PodReadyMetricName)) assert.False(t, decoratedResultMetric.HasField(PodScheduledMetricName)) + assert.False(t, decoratedResultMetric.HasField(PodUnknownMetricName)) } func TestPodStore_addStatus_adds_all_pod_conditions_as_metrics_when_true_false_unknown(t *testing.T) { decoratedResultMetric := runAddStatusToGetDecoratedCIMetric("./test_resources/all_pod_conditions_valid.json", true) - assert.Equal(t, 1, decoratedResultMetric.GetField(PodInitializedMetricName)) assert.Equal(t, 0, decoratedResultMetric.GetField(PodReadyMetricName)) assert.Equal(t, 0, decoratedResultMetric.GetField(PodScheduledMetricName)) + assert.Equal(t, 1, decoratedResultMetric.GetField(PodUnknownMetricName)) +} + +func TestPodStore_addStatus_adds_all_pod_conditions_as_metrics_when_Ready_Scheduled_Condition_Unknown(t *testing.T) { + decoratedResultMetric := runAddStatusToGetDecoratedCIMetric("./test_resources/pod_Ready_Scheduled_Condition_Unknown.json", true) + + assert.Equal(t, 0, decoratedResultMetric.GetField(PodReadyMetricName)) + assert.Equal(t, 0, decoratedResultMetric.GetField(PodScheduledMetricName)) + assert.Equal(t, 1, decoratedResultMetric.GetField(PodUnknownMetricName)) } func TestPodStore_addStatus_adds_all_pod_conditions_as_metrics_when_unexpected(t *testing.T) { decoratedResultMetric := runAddStatusToGetDecoratedCIMetric("./test_resources/one_pod_condition_invalid.json", true) - assert.Equal(t, 0, decoratedResultMetric.GetField(PodInitializedMetricName)) assert.Equal(t, 1, decoratedResultMetric.GetField(PodReadyMetricName)) assert.Equal(t, 1, decoratedResultMetric.GetField(PodScheduledMetricName)) + assert.Equal(t, 0, decoratedResultMetric.GetField(PodUnknownMetricName)) } func TestPodStore_addStatus_enhanced_metrics(t *testing.T) { diff --git a/receiver/awscontainerinsightreceiver/internal/stores/test_resources/pod_Ready_Scheduled_Condition_Unknown.json b/receiver/awscontainerinsightreceiver/internal/stores/test_resources/pod_Ready_Scheduled_Condition_Unknown.json new file mode 100644 index 0000000000000..e496a61f2f083 --- /dev/null +++ b/receiver/awscontainerinsightreceiver/internal/stores/test_resources/pod_Ready_Scheduled_Condition_Unknown.json @@ -0,0 +1,156 @@ +{ + "kind": "PodList", + "apiVersion": "v1", + "metadata": { + + }, + "items": [ + { + "metadata": { + "name": "cpu-limit", + "namespace": "default", + "ownerReferences": [ + { + "apiVersion": "apps/v1", + "blockOwnerDeletion": true, + "controller": true, + "kind": "DaemonSet", + "name": "DaemonSetTest", + "uid": "36779a62-4aca-11e9-977b-0672b6c6fc94" + } + ], + "selfLink": "/api/v1/namespaces/default/pods/cpu-limit", + "uid": "764d01e1-2a2f-11e9-95ea-0a695d7ce286", + "resourceVersion": "5671573", + "creationTimestamp": "2019-02-06T16:51:34Z", + "labels": { + "app": "hello_test" + }, + "annotations": { + "kubernetes.io/config.seen": "2019-02-19T00:06:56.109155665Z", + "kubernetes.io/config.source": "api" + } + }, + "spec": { + "volumes": [ + { + "name": "default-token-tlgw7", + "secret": { + "secretName": "default-token-tlgw7", + "defaultMode": 420 + } + } + ], + "containers": [ + { + "name": "ubuntu", + "image": "ubuntu", + "command": [ + "/bin/bash" + ], + "args": [ + "-c", + "sleep 300000000" + ], + "resources": { + "limits": { + "cpu": "10m", + "memory": "50Mi" + }, + "requests": { + "cpu": "10m", + "memory": "50Mi" + } + }, + "volumeMounts": [ + { + "name": "default-token-tlgw7", + "readOnly": true, + "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" + } + ], + "terminationMessagePath": "/dev/termination-log", + "terminationMessagePolicy": "File", + "imagePullPolicy": "Always" + } + ], + "restartPolicy": "Always", + "terminationGracePeriodSeconds": 30, + "dnsPolicy": "ClusterFirst", + "serviceAccountName": "default", + "serviceAccount": "default", + "nodeName": "ip-192-168-67-127.us-west-2.compute.internal", + "securityContext": { + + }, + "schedulerName": "default-scheduler", + "tolerations": [ + { + "key": "node.kubernetes.io/not-ready", + "operator": "Exists", + "effect": "NoExecute", + "tolerationSeconds": 300 + }, + { + "key": "node.kubernetes.io/unreachable", + "operator": "Exists", + "effect": "NoExecute", + "tolerationSeconds": 300 + } + ], + "priority": 0 + }, + "status": { + "phase": "Running", + "conditions": [ + { + "type": "Initialized", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": "2019-02-06T16:51:34Z" + }, + { + "type": "Ready", + "status": "Unknown", + "lastProbeTime": null, + "lastTransitionTime": "2019-02-06T16:51:43Z" + }, + { + "type": "ContainersReady", + "status": "True", + "lastProbeTime": null, + "lastTransitionTime": null + }, + { + "type": "PodScheduled", + "status": "Unknown", + "lastProbeTime": null, + "lastTransitionTime": "2019-02-06T16:51:34Z" + } + ], + "hostIP": "192.168.67.127", + "podIP": "192.168.76.93", + "startTime": "2019-02-06T16:51:34Z", + "containerStatuses": [ + { + "name": "ubuntu", + "state": { + "running": { + "startedAt": "2019-02-06T16:51:42Z" + } + }, + "lastState": { + + }, + "ready": true, + "restartCount": 0, + "image": "ubuntu:latest", + "imageID": "docker-pullable://ubuntu@sha256:7a47ccc3bbe8a451b500d2b53104868b46d60ee8f5b35a24b41a86077c650210", + "containerID": "docker://637631e2634ea92c0c1aa5d24734cfe794f09c57933026592c12acafbaf6972c" + } + ], + "qosClass": "Guaranteed" + } + } + ] +} \ No newline at end of file