From 3f8eda37ce6345c35223b5c163e82d8a8c0ab51c Mon Sep 17 00:00:00 2001 From: Lu Fengqi Date: Tue, 26 Feb 2019 17:52:03 +0800 Subject: [PATCH] cri_stats_provider: overload nil as 0 for exited containers stats Always report 0 cpu/memory usage for exited containers to make metrics-server work as expect. Signed-off-by: Lu Fengqi --- pkg/kubelet/stats/cri_stats_provider.go | 15 ++++++ pkg/kubelet/stats/cri_stats_provider_test.go | 56 +++++++++++++++++--- pkg/kubelet/stats/helper.go | 4 ++ 3 files changed, 67 insertions(+), 8 deletions(-) diff --git a/pkg/kubelet/stats/cri_stats_provider.go b/pkg/kubelet/stats/cri_stats_provider.go index c2657d9b6a42..3e08609db432 100644 --- a/pkg/kubelet/stats/cri_stats_provider.go +++ b/pkg/kubelet/stats/cri_stats_provider.go @@ -455,12 +455,19 @@ func (p *criStatsProvider) makeContainerStats( if usageNanoCores != nil { result.CPU.UsageNanoCores = usageNanoCores } + } else { + result.CPU.Time = metav1.NewTime(time.Unix(0, time.Now().UnixNano())) + result.CPU.UsageCoreNanoSeconds = Uint64Ptr(0) + result.CPU.UsageNanoCores = Uint64Ptr(0) } if stats.Memory != nil { result.Memory.Time = metav1.NewTime(time.Unix(0, stats.Memory.Timestamp)) if stats.Memory.WorkingSetBytes != nil { result.Memory.WorkingSetBytes = &stats.Memory.WorkingSetBytes.Value } + } else { + result.Memory.Time = metav1.NewTime(time.Unix(0, time.Now().UnixNano())) + result.Memory.WorkingSetBytes = Uint64Ptr(0) } if stats.WritableLayer != nil { result.Rootfs.Time = metav1.NewTime(time.Unix(0, stats.WritableLayer.Timestamp)) @@ -516,13 +523,21 @@ func (p *criStatsProvider) makeContainerCPUAndMemoryStats( if usageNanoCores != nil { result.CPU.UsageNanoCores = usageNanoCores } + } else { + result.CPU.Time = metav1.NewTime(time.Unix(0, time.Now().UnixNano())) + result.CPU.UsageCoreNanoSeconds = Uint64Ptr(0) + result.CPU.UsageNanoCores = Uint64Ptr(0) } if stats.Memory != nil { result.Memory.Time = metav1.NewTime(time.Unix(0, stats.Memory.Timestamp)) if stats.Memory.WorkingSetBytes != nil { result.Memory.WorkingSetBytes = &stats.Memory.WorkingSetBytes.Value } + } else { + result.Memory.Time = metav1.NewTime(time.Unix(0, time.Now().UnixNano())) + result.Memory.WorkingSetBytes = Uint64Ptr(0) } + return result } diff --git a/pkg/kubelet/stats/cri_stats_provider_test.go b/pkg/kubelet/stats/cri_stats_provider_test.go index f420f3790561..2c4de8e93fa7 100644 --- a/pkg/kubelet/stats/cri_stats_provider_test.go +++ b/pkg/kubelet/stats/cri_stats_provider_test.go @@ -57,12 +57,15 @@ const ( seedContainer2 = 5000 seedSandbox2 = 6000 seedContainer3 = 7000 + seedSandbox3 = 8000 + seedContainer5 = 9000 ) const ( pName0 = "pod0" pName1 = "pod1" pName2 = "pod2" + pName3 = "pod3" ) const ( @@ -70,6 +73,7 @@ const ( cName1 = "container1-name" cName2 = "container2-name" cName3 = "container3-name" + cName5 = "container5-name" ) func TestCRIListPodStats(t *testing.T) { @@ -101,6 +105,11 @@ func TestCRIListPodStats(t *testing.T) { container4 = makeFakeContainer(sandbox2, cName3, 1, false) containerStats4 = makeFakeContainerStats(container4, imageFsMountpoint) containerLogStats4 = makeFakeLogStats(4000) + + sandbox3 = makeFakePodSandbox("sandbox3-name", "sandbox3-uid", "sandbox3-ns") + container5 = makeFakeContainer(sandbox3, cName5, 0, true) + containerStats5 = makeFakeContainerStats(container5, imageFsMountpoint) + containerLogStats5 = makeFakeLogStats(5000) ) var ( @@ -140,13 +149,13 @@ func TestCRIListPodStats(t *testing.T) { On("GetDirFsInfo", imageFsMountpoint).Return(imageFsInfo, nil). On("GetDirFsInfo", unknownMountpoint).Return(cadvisorapiv2.FsInfo{}, cadvisorfs.ErrNoSuchDevice) fakeRuntimeService.SetFakeSandboxes([]*critest.FakePodSandbox{ - sandbox0, sandbox1, sandbox2, + sandbox0, sandbox1, sandbox2, sandbox3, }) fakeRuntimeService.SetFakeContainers([]*critest.FakeContainer{ - container0, container1, container2, container3, container4, + container0, container1, container2, container3, container4, container5, }) fakeRuntimeService.SetFakeContainerStats([]*runtimeapi.ContainerStats{ - containerStats0, containerStats1, containerStats2, containerStats3, containerStats4, + containerStats0, containerStats1, containerStats2, containerStats3, containerStats4, containerStats5, }) ephemeralVolumes := makeFakeVolumeStats([]string{"ephVolume1, ephVolumes2"}) @@ -161,6 +170,7 @@ func TestCRIListPodStats(t *testing.T) { kuberuntime.BuildContainerLogsDirectory(types.UID("sandbox0-uid"), cName1): containerLogStats1, kuberuntime.BuildContainerLogsDirectory(types.UID("sandbox1-uid"), cName2): containerLogStats2, kuberuntime.BuildContainerLogsDirectory(types.UID("sandbox2-uid"), cName3): containerLogStats4, + kuberuntime.BuildContainerLogsDirectory(types.UID("sandbox3-uid"), cName5): containerLogStats5, } fakeLogStatsProvider := NewFakeLogMetricsService(fakeLogStats) @@ -177,7 +187,7 @@ func TestCRIListPodStats(t *testing.T) { stats, err := provider.ListPodStats() assert := assert.New(t) assert.NoError(err) - assert.Equal(3, len(stats)) + assert.Equal(4, len(stats)) podStatsMap := make(map[statsapi.PodReference]statsapi.PodStats) for _, s := range stats { @@ -239,6 +249,19 @@ func TestCRIListPodStats(t *testing.T) { checkCRINetworkStats(assert, p2.Network, infos[sandbox2.PodSandboxStatus.Id].Stats[0].Network) checkCRIPodCPUAndMemoryStats(assert, p2, infos[sandbox2Cgroup].Stats[0]) + p3 := podStatsMap[statsapi.PodReference{Name: "sandbox3-name", UID: "sandbox3-uid", Namespace: "sandbox3-ns"}] + assert.Equal(sandbox3.CreatedAt, p3.StartTime.UnixNano()) + assert.Equal(1, len(p3.Containers)) + + c5 := p3.Containers[0] + assert.Equal(cName5, c5.Name) + assert.Equal(container5.CreatedAt, c5.StartTime.UnixNano()) + assert.NotNil(c5.CPU.Time) + assert.Zero(*c5.CPU.UsageCoreNanoSeconds) + assert.Zero(*c5.CPU.UsageNanoCores) + assert.NotNil(c5.Memory.Time) + assert.Zero(*c5.Memory.WorkingSetBytes) + mockCadvisor.AssertExpectations(t) } @@ -266,6 +289,10 @@ func TestCRIListPodCPUAndMemoryStats(t *testing.T) { containerStats3 = makeFakeContainerStats(container3, imageFsMountpoint) container4 = makeFakeContainer(sandbox2, cName3, 1, false) containerStats4 = makeFakeContainerStats(container4, imageFsMountpoint) + + sandbox3 = makeFakePodSandbox("sandbox3-name", "sandbox3-uid", "sandbox3-ns") + container5 = makeFakeContainer(sandbox3, cName5, 0, true) + containerStats5 = makeFakeContainerStats(container5, imageFsMountpoint) ) var ( @@ -301,13 +328,13 @@ func TestCRIListPodCPUAndMemoryStats(t *testing.T) { mockCadvisor. On("ContainerInfoV2", "/", options).Return(infos, nil) fakeRuntimeService.SetFakeSandboxes([]*critest.FakePodSandbox{ - sandbox0, sandbox1, sandbox2, + sandbox0, sandbox1, sandbox2, sandbox3, }) fakeRuntimeService.SetFakeContainers([]*critest.FakeContainer{ - container0, container1, container2, container3, container4, + container0, container1, container2, container3, container4, container5, }) fakeRuntimeService.SetFakeContainerStats([]*runtimeapi.ContainerStats{ - containerStats0, containerStats1, containerStats2, containerStats3, containerStats4, + containerStats0, containerStats1, containerStats2, containerStats3, containerStats4, containerStats5, }) ephemeralVolumes := makeFakeVolumeStats([]string{"ephVolume1, ephVolumes2"}) @@ -330,7 +357,7 @@ func TestCRIListPodCPUAndMemoryStats(t *testing.T) { stats, err := provider.ListPodCPUAndMemoryStats() assert := assert.New(t) assert.NoError(err) - assert.Equal(3, len(stats)) + assert.Equal(4, len(stats)) podStatsMap := make(map[statsapi.PodReference]statsapi.PodStats) for _, s := range stats { @@ -399,6 +426,19 @@ func TestCRIListPodCPUAndMemoryStats(t *testing.T) { assert.Nil(c2.Accelerators) assert.Nil(c2.UserDefinedMetrics) + p3 := podStatsMap[statsapi.PodReference{Name: "sandbox3-name", UID: "sandbox3-uid", Namespace: "sandbox3-ns"}] + assert.Equal(sandbox3.CreatedAt, p3.StartTime.UnixNano()) + assert.Equal(1, len(p3.Containers)) + + c5 := p3.Containers[0] + assert.Equal(cName5, c5.Name) + assert.Equal(container5.CreatedAt, c5.StartTime.UnixNano()) + assert.NotNil(c5.CPU.Time) + assert.Zero(*c5.CPU.UsageCoreNanoSeconds) + assert.Zero(*c5.CPU.UsageNanoCores) + assert.NotNil(c5.Memory.Time) + assert.Zero(*c5.Memory.WorkingSetBytes) + mockCadvisor.AssertExpectations(t) } diff --git a/pkg/kubelet/stats/helper.go b/pkg/kubelet/stats/helper.go index 54f3093e5530..a26790a5ec20 100644 --- a/pkg/kubelet/stats/helper.go +++ b/pkg/kubelet/stats/helper.go @@ -326,3 +326,7 @@ func getUint64Value(value *uint64) uint64 { return *value } + +func Uint64Ptr(i uint64) *uint64 { + return &i +}