Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add memory available to summary stats provider #23948

Merged
merged 1 commit into from
Apr 17, 2016
Merged

Add memory available to summary stats provider #23948

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
5 changes: 4 additions & 1 deletion pkg/kubelet/api/v1alpha1/stats/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,10 +128,13 @@ type CPUStats struct {
type MemoryStats struct {
// The time at which these stats were updated.
Time unversioned.Time `json:"time"`
// Available memory for use. This is defined as the memory limit - workingSetBytes.
// If memory limit is undefined, the available bytes is omitted.
AvailableBytes *uint64 `json:"availableBytes,omitempty"`
// Total memory in use. This includes all memory regardless of when it was accessed.
UsageBytes *uint64 `json:"usageBytes,omitempty"`
// The amount of working set memory. This includes recently accessed memory,
// dirty memory, and kernel memory. UsageBytes is <= TotalBytes.
// dirty memory, and kernel memory. WorkingSetBytes is <= UsageBytes
WorkingSetBytes *uint64 `json:"workingSetBytes,omitempty"`
// Cumulative number of minor page faults.
PageFaults *uint64 `json:"pageFaults,omitempty"`
Expand Down
16 changes: 16 additions & 0 deletions pkg/kubelet/server/stats/summary.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"k8s.io/kubernetes/pkg/types"

"github.com/golang/glog"

cadvisorapiv1 "github.com/google/cadvisor/info/v1"
cadvisorapiv2 "github.com/google/cadvisor/info/v2"
)
Expand Down Expand Up @@ -283,12 +284,27 @@ func (sb *summaryBuilder) containerInfoV2ToStats(
PageFaults: &pageFaults,
MajorPageFaults: &majorPageFaults,
}
// availableBytes = memory limit (if known) - workingset
if !isMemoryUnlimited(info.Spec.Memory.Limit) {
availableBytes := info.Spec.Memory.Limit - cstat.Memory.WorkingSet
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does memory limit include swap space? If not, couldn't availableBytes end up being negative?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We do not support swap as of now. If/when we do, memory limit would not include swap, we would have to use the total limit - swap + ram

cStats.Memory.AvailableBytes = &availableBytes
}
}

sb.containerInfoV2FsStats(info, &cStats)
cStats.UserDefinedMetrics = sb.containerInfoV2ToUserDefinedMetrics(info)
return cStats
}

// Size after which we consider memory to be "unlimited". This is not
// MaxInt64 due to rounding by the kernel.
// TODO: cadvisor should export this https://github.com/google/cadvisor/blob/master/metrics/prometheus.go#L596
const maxMemorySize = uint64(1 << 62)

func isMemoryUnlimited(v uint64) bool {
return v > maxMemorySize
}

func (sb *summaryBuilder) containerInfoV2ToNetworkStats(name string, info *cadvisorapiv2.ContainerInfo) *stats.NetworkStats {
if !info.Spec.HasNetwork {
return nil
Expand Down
71 changes: 51 additions & 20 deletions pkg/kubelet/server/stats/summary_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ func TestBuildSummary(t *testing.T) {
// Pod0 - Namespace0
"/pod0-i": summaryTestContainerInfo(seedPod0Infra, pName0, namespace0, leaky.PodInfraContainerName),
"/pod0-c0": summaryTestContainerInfo(seedPod0Container0, pName0, namespace0, cName00),
"/pod0-c2": summaryTestContainerInfo(seedPod0Container1, pName0, namespace0, cName01),
"/pod0-c1": summaryTestContainerInfo(seedPod0Container1, pName0, namespace0, cName01),
// Pod1 - Namespace0
"/pod1-i": summaryTestContainerInfo(seedPod1Infra, pName1, namespace0, leaky.PodInfraContainerName),
"/pod1-c0": summaryTestContainerInfo(seedPod1Container, pName1, namespace0, cName10),
Expand All @@ -111,6 +111,20 @@ func TestBuildSummary(t *testing.T) {
rootfs := v2.FsInfo{}
imagefs := v2.FsInfo{}

// memory limit overrides for each container (used to test available bytes if a memory limit is known)
memoryLimitOverrides := map[string]uint64{
"/": uint64(1 << 30),
"/pod2-c0": uint64(1 << 15),
}
for name, memoryLimitOverride := range memoryLimitOverrides {
info, found := infos[name]
if !found {
t.Errorf("No container defined with name %v", name)
}
info.Spec.Memory.Limit = memoryLimitOverride
infos[name] = info
}

sb := &summaryBuilder{
newFsResourceAnalyzer(&MockStatsProvider{}, time.Minute*5), &node, nodeConfig, rootfs, imagefs, infos}
summary, err := sb.build()
Expand All @@ -120,23 +134,29 @@ func TestBuildSummary(t *testing.T) {
assert.Equal(t, "FooNode", nodeStats.NodeName)
assert.EqualValues(t, testTime(creationTime, seedRoot).Unix(), nodeStats.StartTime.Time.Unix())
checkCPUStats(t, "Node", seedRoot, nodeStats.CPU)
checkMemoryStats(t, "Node", seedRoot, nodeStats.Memory)
checkMemoryStats(t, "Node", seedRoot, infos["/"], nodeStats.Memory)
checkNetworkStats(t, "Node", seedRoot, nodeStats.Network)

systemSeeds := map[string]int{
kubestats.SystemContainerRuntime: seedRuntime,
kubestats.SystemContainerKubelet: seedKubelet,
kubestats.SystemContainerMisc: seedMisc,
}
systemContainerToNodeCgroup := map[string]string{
kubestats.SystemContainerRuntime: nodeConfig.RuntimeCgroupsName,
kubestats.SystemContainerKubelet: nodeConfig.KubeletCgroupsName,
kubestats.SystemContainerMisc: nodeConfig.SystemCgroupsName,
}
for _, sys := range nodeStats.SystemContainers {
name := sys.Name
info := infos[systemContainerToNodeCgroup[name]]
seed, found := systemSeeds[name]
if !found {
t.Errorf("Unknown SystemContainer: %q", name)
}
assert.EqualValues(t, testTime(creationTime, seed).Unix(), sys.StartTime.Time.Unix(), name+".StartTime")
checkCPUStats(t, name, seed, sys.CPU)
checkMemoryStats(t, name, seed, sys.Memory)
checkMemoryStats(t, name, seed, info, sys.Memory)
}

assert.Equal(t, 3, len(summary.Pods))
Expand All @@ -155,36 +175,36 @@ func TestBuildSummary(t *testing.T) {
}
con := indexCon[cName00]
assert.EqualValues(t, testTime(creationTime, seedPod0Container0).Unix(), con.StartTime.Time.Unix())
checkCPUStats(t, "container", seedPod0Container0, con.CPU)
checkMemoryStats(t, "container", seedPod0Container0, con.Memory)
checkCPUStats(t, "Pod0Container0", seedPod0Container0, con.CPU)
checkMemoryStats(t, "Pod0Conainer0", seedPod0Container0, infos["/pod0-c0"], con.Memory)

con = indexCon[cName01]
assert.EqualValues(t, testTime(creationTime, seedPod0Container1).Unix(), con.StartTime.Time.Unix())
checkCPUStats(t, "container", seedPod0Container1, con.CPU)
checkMemoryStats(t, "container", seedPod0Container1, con.Memory)
checkCPUStats(t, "Pod0Container1", seedPod0Container1, con.CPU)
checkMemoryStats(t, "Pod0Container1", seedPod0Container1, infos["/pod0-c1"], con.Memory)

assert.EqualValues(t, testTime(creationTime, seedPod0Infra).Unix(), ps.StartTime.Time.Unix())
checkNetworkStats(t, "Pod", seedPod0Infra, ps.Network)
checkNetworkStats(t, "Pod0", seedPod0Infra, ps.Network)

// Validate Pod1 Results
ps, found = indexPods[prf1]
assert.True(t, found)
assert.Len(t, ps.Containers, 1)
con = ps.Containers[0]
assert.Equal(t, cName10, con.Name)
checkCPUStats(t, "container", seedPod1Container, con.CPU)
checkMemoryStats(t, "container", seedPod1Container, con.Memory)
checkNetworkStats(t, "Pod", seedPod1Infra, ps.Network)
checkCPUStats(t, "Pod1Container0", seedPod1Container, con.CPU)
checkMemoryStats(t, "Pod1Container0", seedPod1Container, infos["/pod1-c0"], con.Memory)
checkNetworkStats(t, "Pod1", seedPod1Infra, ps.Network)

// Validate Pod2 Results
ps, found = indexPods[prf2]
assert.True(t, found)
assert.Len(t, ps.Containers, 1)
con = ps.Containers[0]
assert.Equal(t, cName20, con.Name)
checkCPUStats(t, "container", seedPod2Container, con.CPU)
checkMemoryStats(t, "container", seedPod2Container, con.Memory)
checkNetworkStats(t, "Pod", seedPod2Infra, ps.Network)
checkCPUStats(t, "Pod2Container0", seedPod2Container, con.CPU)
checkMemoryStats(t, "Pod2Container0", seedPod2Container, infos["/pod2-c0"], con.Memory)
checkNetworkStats(t, "Pod2", seedPod2Infra, ps.Network)
}

func generateCustomMetricSpec() []v1.MetricSpec {
Expand Down Expand Up @@ -242,12 +262,17 @@ func summaryTestContainerInfo(seed int, podName string, podNamespace string, con
"io.kubernetes.container.name": containerName,
}
}
// by default, kernel will set memory.limit_in_bytes to 1 << 63 if not bounded
unlimitedMemory := uint64(1 << 63)
spec := v2.ContainerSpec{
CreationTime: testTime(creationTime, seed),
HasCpu: true,
HasMemory: true,
HasNetwork: true,
Labels: labels,
CreationTime: testTime(creationTime, seed),
HasCpu: true,
HasMemory: true,
HasNetwork: true,
Labels: labels,
Memory: v2.MemorySpec{
Limit: unlimitedMemory,
},
CustomMetrics: generateCustomMetricSpec(),
}

Expand Down Expand Up @@ -306,12 +331,18 @@ func checkCPUStats(t *testing.T, label string, seed int, stats *kubestats.CPUSta
assert.EqualValues(t, seed+offsetCPUUsageCoreSeconds, *stats.UsageCoreNanoSeconds, label+".CPU.UsageCoreSeconds")
}

func checkMemoryStats(t *testing.T, label string, seed int, stats *kubestats.MemoryStats) {
func checkMemoryStats(t *testing.T, label string, seed int, info v2.ContainerInfo, stats *kubestats.MemoryStats) {
assert.EqualValues(t, testTime(timestamp, seed).Unix(), stats.Time.Time.Unix(), label+".Mem.Time")
assert.EqualValues(t, seed+offsetMemUsageBytes, *stats.UsageBytes, label+".Mem.UsageBytes")
assert.EqualValues(t, seed+offsetMemWorkingSetBytes, *stats.WorkingSetBytes, label+".Mem.WorkingSetBytes")
assert.EqualValues(t, seed+offsetMemPageFaults, *stats.PageFaults, label+".Mem.PageFaults")
assert.EqualValues(t, seed+offsetMemMajorPageFaults, *stats.MajorPageFaults, label+".Mem.MajorPageFaults")
if !info.Spec.HasMemory || isMemoryUnlimited(info.Spec.Memory.Limit) {
assert.Nil(t, stats.AvailableBytes, label+".Mem.AvailableBytes")
} else {
expected := info.Spec.Memory.Limit - *stats.WorkingSetBytes
assert.EqualValues(t, expected, *stats.AvailableBytes, label+".Mem.AvailableBytes")
}
}

func TestCustomMetrics(t *testing.T) {
Expand Down