/
container_info_processor.go
208 lines (178 loc) · 7.78 KB
/
container_info_processor.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
package cadvisor
import (
"log"
"path"
"strconv"
"strings"
common "github.com/aws-observability/aws-otel-collector/internal/aws/containerinsightcommon"
"github.com/aws-observability/aws-otel-collector/internal/receiver/awscontainerinsightreceiver/cadvisor/extractors"
"github.com/aws-observability/aws-otel-collector/internal/receiver/awscontainerinsightreceiver/host"
cinfo "github.com/google/cadvisor/info/v1"
"go.uber.org/zap"
)
const (
// TODO: https://github.com/containerd/cri/issues/922#issuecomment-423729537 the container name can be empty on containerd
infraContainerName = "POD"
podNameLabel = "io.kubernetes.pod.name"
namespaceLabel = "io.kubernetes.pod.namespace"
podIdLabel = "io.kubernetes.pod.uid"
containerNameLabel = "io.kubernetes.container.name"
)
// podKey contains information of a pod extracted from containers owned by it.
// containers has label information for a pod and their cgroup path is one level deeper.
type podKey struct {
cgroupPath string
podId string
containerIds []string
podName string
namespace string
}
func processContainers(cInfos []*cinfo.ContainerInfo, mInfo *host.MachineInfo, containerOrchestrator string, logger *zap.Logger) []*extractors.CAdvisorMetric {
var metrics []*extractors.CAdvisorMetric
podKeys := make(map[string]podKey)
for _, cInfo := range cInfos {
if len(cInfo.Stats) == 0 {
continue
}
outMetrics, outPodKey := processContainer(cInfo, mInfo, containerOrchestrator, logger)
metrics = append(metrics, outMetrics...)
// Save pod cgroup path we collected from containers under it.
if outPodKey != nil {
if podKey, ok := podKeys[outPodKey.cgroupPath]; !ok {
podKeys[outPodKey.cgroupPath] = *outPodKey
} else {
//collect the container ids associated with a pod
podKey.containerIds = append(podKey.containerIds, outPodKey.containerIds...)
}
}
}
beforePod := len(metrics)
for _, cInfo := range cInfos {
if len(cInfo.Stats) == 0 {
continue
}
metrics = append(metrics, processPod(cInfo, mInfo, podKeys, logger)...)
}
// This happens when our cgroup path based pod detection logic is not working.
if len(metrics) == beforePod {
logger.Warn("No pod metric collected", zap.Any("metrics count", beforePod))
}
metrics = mergeMetrics(metrics)
// now := time.Now()
// for _, extractor := range MetricsExtractors {
// extractor.CleanUp(now)
// }
return metrics
}
// processContainers get metrics for individual container and gather information for pod so we can look it up later.
func processContainer(info *cinfo.ContainerInfo, mInfo *host.MachineInfo, containerOrchestrator string, logger *zap.Logger) ([]*extractors.CAdvisorMetric, *podKey) {
var result []*extractors.CAdvisorMetric
var pKey *podKey
if isContainerInContainer(info.Name) {
logger.Warn("drop metric because it's nested container", zap.Any("name", info.Name))
return result, pKey
}
tags := map[string]string{}
var containerType string
if info.Name != "/" {
// Only a container has all these three labels set.
containerName := info.Spec.Labels[containerNameLabel]
namespace := info.Spec.Labels[namespaceLabel]
podName := info.Spec.Labels[podNameLabel]
podId := info.Spec.Labels[podIdLabel]
if containerName == "" || namespace == "" || podName == "" {
logger.Debug("Container labels are missing",
zap.String("containerName", containerName),
zap.String("namespace", namespace),
zap.String("podName", podName),
zap.String("podId", podId),
)
return result, pKey
}
// Pod's cgroup path is parent for a container.
// contianer name: /kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod04d39715_075e_4c7c_b128_67f7897c05b7.slice/docker-57b3dabd69b94beb462244a0c15c244b509adad0940cdcc67ca079b8208ec1f2.scope
// pod name: /kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod04d39715_075e_4c7c_b128_67f7897c05b7.slice/
podPath := path.Dir(info.Name)
pKey = &podKey{cgroupPath: podPath, podName: podName, podId: podId, namespace: namespace}
tags[common.PodIdKey] = podId
tags[common.K8sPodNameKey] = podName
tags[common.K8sNamespace] = namespace
if containerName != infraContainerName {
tags[common.ContainerNamekey] = containerName
containerId := path.Base(info.Name)
tags[common.ContainerIdkey] = containerId
pKey.containerIds = []string{containerId}
containerType = common.TypeContainer
} else {
// NOTE: the pod here is only used by NetMetricExtractor,
// other pod info like CPU, Mem are dealt within in processPod.
containerType = common.TypePod
}
} else {
containerType = common.TypeNode
if containerOrchestrator == common.ECS {
containerType = common.TypeInstance
}
}
tags[common.Timestamp] = strconv.FormatInt(extractors.GetStats(info).Timestamp.UnixNano(), 10)
for _, extractor := range GetMetricsExtractors() {
if extractor.HasValue(info) {
result = append(result, extractor.GetValue(info, mInfo, containerType)...)
}
}
for _, ele := range result {
ele.AddTags(tags)
}
return result, pKey
}
func processPod(info *cinfo.ContainerInfo, mInfo *host.MachineInfo, podKeys map[string]podKey, logger *zap.Logger) []*extractors.CAdvisorMetric {
var result []*extractors.CAdvisorMetric
if isContainerInContainer(info.Name) {
log.Printf("D! drop metric because it's nested container, name %s", info.Name)
return result
}
podKey := getPodKey(info, podKeys)
if podKey == nil {
return result
}
tags := map[string]string{}
tags[common.PodIdKey] = podKey.podId
tags[common.K8sPodNameKey] = podKey.podName
tags[common.K8sNamespace] = podKey.namespace
tags[common.Timestamp] = strconv.FormatInt(extractors.GetStats(info).Timestamp.UnixNano(), 10)
for _, extractor := range GetMetricsExtractors() {
if extractor.HasValue(info) {
result = append(result, extractor.GetValue(info, mInfo, common.TypePod)...)
}
}
for _, ele := range result {
ele.AddTags(tags)
}
return result
}
func getPodKey(info *cinfo.ContainerInfo, podKeys map[string]podKey) *podKey {
key := info.Name
if v, ok := podKeys[key]; ok {
return &v
}
return nil
}
// Check if it's a container running inside container, caller will drop the metric when return value is true.
// The validation is based on ContainerReference.Name, which is essentially cgroup path.
// The first version is from https://github.com/aws/amazon-cloudwatch-agent/commit/e8daa5f5926c5a5f38e0ceb746c141be463e11e4#diff-599185154c116b295172b56311729990d20672f6659500870997c018ce072100
// But the logic no longer works when docker is using systemd as cgroup driver, because a prefix like `kubepods` is attached to each segment.
// The new name pattern with systemd is
// - Guaranteed /kubepods.slice/kubepods-podc8f7bb69_65f2_4b61_ae5a_9b19ac47a239.slice/docker-523b624a86a2a74c2bedf586d8448c86887ef7858a8dec037d6559e5ad3fccb5.scope
// - Burstable /kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podab0e310c_0bdb_48e8_ac87_81a701514645.slice/docker-caa8a5e51cd6610f8f0110b491e8187d23488b9635acccf0355a7975fd3ff158.scope
// - Docker in Docker /kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podc9adcee4_c874_4dad_8bc8_accdbd67ac3a.slice/docker-e58cfbc8b67f6e1af458efdd31cb2a8abdbf9f95db64f4c852b701285a09d40e.scope/docker/fb651068cfbd4bf3d45fb092ec9451f8d1a36b3753687bbaa0a9920617eae5b9
// So we check the number of segements within the cgroup path to determine if it's a container running in container.
func isContainerInContainer(p string) bool {
segs := strings.Split(strings.TrimLeft(p, "/"), "/")
// Without nested container, the number of segments (regardless of cgroupfs/systemd) are either 3 or 4 (depends on QoS)
// /kubepods/pod_id/docker_id
// /kubepods/qos/pod_id/docker_id
// With nested container, the number of segments are either 5 or 6
// /kubepods/pod_id/docker_id/docker/docker_id
// /kubepods/qos/pod_id/docker_id/docker/docker_id
return len(segs) > 4
}