This repository has been archived by the owner on Jul 1, 2023. It is now read-only.
forked from kubernetes/kubernetes
-
Notifications
You must be signed in to change notification settings - Fork 1
/
helper.go
255 lines (231 loc) · 7.78 KB
/
helper.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package stats
import (
"fmt"
"time"
"github.com/golang/glog"
cadvisorapiv1 "github.com/google/cadvisor/info/v1"
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
"k8s.io/kubernetes/pkg/kubelet/cadvisor"
"k8s.io/kubernetes/pkg/kubelet/network"
)
// cadvisorInfoToContainerStats returns the statsapi.ContainerStats converted
// from the container and filesystem info.
func cadvisorInfoToContainerStats(name string, info *cadvisorapiv2.ContainerInfo, rootFs, imageFs *cadvisorapiv2.FsInfo) *statsapi.ContainerStats {
result := &statsapi.ContainerStats{
StartTime: metav1.NewTime(info.Spec.CreationTime),
Name: name,
}
cstat, found := latestContainerStats(info)
if !found {
return result
}
if info.Spec.HasCpu {
cpuStats := statsapi.CPUStats{
Time: metav1.NewTime(cstat.Timestamp),
}
if cstat.CpuInst != nil {
cpuStats.UsageNanoCores = &cstat.CpuInst.Usage.Total
}
if cstat.Cpu != nil {
cpuStats.UsageCoreNanoSeconds = &cstat.Cpu.Usage.Total
}
result.CPU = &cpuStats
}
if info.Spec.HasMemory {
pageFaults := cstat.Memory.ContainerData.Pgfault
majorPageFaults := cstat.Memory.ContainerData.Pgmajfault
result.Memory = &statsapi.MemoryStats{
Time: metav1.NewTime(cstat.Timestamp),
UsageBytes: &cstat.Memory.Usage,
WorkingSetBytes: &cstat.Memory.WorkingSet,
RSSBytes: &cstat.Memory.RSS,
PageFaults: &pageFaults,
MajorPageFaults: &majorPageFaults,
}
// availableBytes = memory limit (if known) - workingset
if !isMemoryUnlimited(info.Spec.Memory.Limit) {
availableBytes := info.Spec.Memory.Limit - cstat.Memory.WorkingSet
result.Memory.AvailableBytes = &availableBytes
}
}
if rootFs != nil {
// The container logs live on the node rootfs device
result.Logs = &statsapi.FsStats{
Time: metav1.NewTime(cstat.Timestamp),
AvailableBytes: &rootFs.Available,
CapacityBytes: &rootFs.Capacity,
InodesFree: rootFs.InodesFree,
Inodes: rootFs.Inodes,
}
if rootFs.Inodes != nil && rootFs.InodesFree != nil {
logsInodesUsed := *rootFs.Inodes - *rootFs.InodesFree
result.Logs.InodesUsed = &logsInodesUsed
}
}
if imageFs != nil {
// The container rootFs lives on the imageFs devices (which may not be the node root fs)
result.Rootfs = &statsapi.FsStats{
Time: metav1.NewTime(cstat.Timestamp),
AvailableBytes: &imageFs.Available,
CapacityBytes: &imageFs.Capacity,
InodesFree: imageFs.InodesFree,
Inodes: imageFs.Inodes,
}
}
cfs := cstat.Filesystem
if cfs != nil {
if cfs.BaseUsageBytes != nil {
if result.Rootfs != nil {
rootfsUsage := *cfs.BaseUsageBytes
result.Rootfs.UsedBytes = &rootfsUsage
}
if cfs.TotalUsageBytes != nil && result.Logs != nil {
logsUsage := *cfs.TotalUsageBytes - *cfs.BaseUsageBytes
result.Logs.UsedBytes = &logsUsage
}
}
if cfs.InodeUsage != nil && result.Rootfs != nil {
rootInodes := *cfs.InodeUsage
result.Rootfs.InodesUsed = &rootInodes
}
}
result.UserDefinedMetrics = cadvisorInfoToUserDefinedMetrics(info)
return result
}
// cadvisorInfoToNetworkStats returns the statsapi.NetworkStats converted from
// the container info from cadvisor.
func cadvisorInfoToNetworkStats(name string, info *cadvisorapiv2.ContainerInfo) *statsapi.NetworkStats {
if !info.Spec.HasNetwork {
return nil
}
cstat, found := latestContainerStats(info)
if !found {
return nil
}
for _, inter := range cstat.Network.Interfaces {
if inter.Name == network.DefaultInterfaceName {
return &statsapi.NetworkStats{
Time: metav1.NewTime(cstat.Timestamp),
RxBytes: &inter.RxBytes,
RxErrors: &inter.RxErrors,
TxBytes: &inter.TxBytes,
TxErrors: &inter.TxErrors,
}
}
}
glog.V(4).Infof("Missing default interface %q for %s", network.DefaultInterfaceName, name)
return nil
}
// cadvisorInfoToUserDefinedMetrics returns the statsapi.UserDefinedMetric
// converted from the container info from cadvisor.
func cadvisorInfoToUserDefinedMetrics(info *cadvisorapiv2.ContainerInfo) []statsapi.UserDefinedMetric {
type specVal struct {
ref statsapi.UserDefinedMetricDescriptor
valType cadvisorapiv1.DataType
time time.Time
value float64
}
udmMap := map[string]*specVal{}
for _, spec := range info.Spec.CustomMetrics {
udmMap[spec.Name] = &specVal{
ref: statsapi.UserDefinedMetricDescriptor{
Name: spec.Name,
Type: statsapi.UserDefinedMetricType(spec.Type),
Units: spec.Units,
},
valType: spec.Format,
}
}
for _, stat := range info.Stats {
for name, values := range stat.CustomMetrics {
specVal, ok := udmMap[name]
if !ok {
glog.Warningf("spec for custom metric %q is missing from cAdvisor output. Spec: %+v, Metrics: %+v", name, info.Spec, stat.CustomMetrics)
continue
}
for _, value := range values {
// Pick the most recent value
if value.Timestamp.Before(specVal.time) {
continue
}
specVal.time = value.Timestamp
specVal.value = value.FloatValue
if specVal.valType == cadvisorapiv1.IntType {
specVal.value = float64(value.IntValue)
}
}
}
}
var udm []statsapi.UserDefinedMetric
for _, specVal := range udmMap {
udm = append(udm, statsapi.UserDefinedMetric{
UserDefinedMetricDescriptor: specVal.ref,
Time: metav1.NewTime(specVal.time),
Value: specVal.value,
})
}
return udm
}
// latestContainerStats returns the latest container stats from cadvisor, or nil if none exist
func latestContainerStats(info *cadvisorapiv2.ContainerInfo) (*cadvisorapiv2.ContainerStats, bool) {
stats := info.Stats
if len(stats) < 1 {
return nil, false
}
latest := stats[len(stats)-1]
if latest == nil {
return nil, false
}
return latest, true
}
func isMemoryUnlimited(v uint64) bool {
// Size after which we consider memory to be "unlimited". This is not
// MaxInt64 due to rounding by the kernel.
// TODO: cadvisor should export this https://github.com/google/cadvisor/blob/master/metrics/prometheus.go#L596
const maxMemorySize = uint64(1 << 62)
return v > maxMemorySize
}
// getCgroupInfo returns the information of the container with the specified
// containerName from cadvisor.
func getCgroupInfo(cadvisor cadvisor.Interface, containerName string) (*cadvisorapiv2.ContainerInfo, error) {
infoMap, err := cadvisor.ContainerInfoV2(containerName, cadvisorapiv2.RequestOptions{
IdType: cadvisorapiv2.TypeName,
Count: 2, // 2 samples are needed to compute "instantaneous" CPU
Recursive: false,
})
if err != nil {
return nil, fmt.Errorf("failed to get container info for %q: %v", containerName, err)
}
if len(infoMap) != 1 {
return nil, fmt.Errorf("unexpected number of containers: %v", len(infoMap))
}
info := infoMap[containerName]
return &info, nil
}
// getCgroupStats returns the latest stats of the container having the
// specified containerName from cadvisor.
func getCgroupStats(cadvisor cadvisor.Interface, containerName string) (*cadvisorapiv2.ContainerStats, error) {
info, err := getCgroupInfo(cadvisor, containerName)
if err != nil {
return nil, err
}
stats, found := latestContainerStats(info)
if !found {
return nil, fmt.Errorf("failed to get latest stats from container info for %q", containerName)
}
return stats, nil
}