Skip to content
Permalink
Browse files

Merge pull request #72005 from dashpole/automated-cherry-pick-of-#595…

…25-upstream-release-1.11

Automated cherry pick of #59525: fix node and kubelet start times
  • Loading branch information...
k8s-ci-robot committed Mar 11, 2019
2 parents b2539d5 + 395daae commit ede55fd572985547208c79eb73c122f3e8f7f79c
@@ -16,6 +16,7 @@ go_library(
"//pkg/kubelet/apis/stats/v1alpha1:go_default_library",
"//pkg/kubelet/cm:go_default_library",
"//pkg/kubelet/container:go_default_library",
"//pkg/kubelet/util:go_default_library",
"//pkg/kubelet/util/format:go_default_library",
"//pkg/volume:go_default_library",
"//vendor/github.com/emicklei/go-restful:go_default_library",
@@ -21,7 +21,9 @@ import (

"github.com/golang/glog"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
statsapi "k8s.io/kubernetes/pkg/kubelet/apis/stats/v1alpha1"
"k8s.io/kubernetes/pkg/kubelet/util"
)

type SummaryProvider interface {
@@ -32,6 +34,11 @@ type SummaryProvider interface {

// summaryProviderImpl implements the SummaryProvider interface.
type summaryProviderImpl struct {
// kubeletCreationTime is the time at which the summaryProvider was created.
kubeletCreationTime metav1.Time
// systemBootTime is the time at which the system was started
systemBootTime metav1.Time

provider StatsProvider
}

@@ -40,7 +47,18 @@ var _ SummaryProvider = &summaryProviderImpl{}
// NewSummaryProvider returns a SummaryProvider using the stats provided by the
// specified statsProvider.
func NewSummaryProvider(statsProvider StatsProvider) SummaryProvider {
return &summaryProviderImpl{statsProvider}
kubeletCreationTime := metav1.Now()
bootTime, err := util.GetBootTime()
if err != nil {
// bootTime will be zero if we encounter an error getting the boot time.
glog.Warningf("Error getting system boot time. Node metrics will have an incorrect start time: %v", err)
}

return &summaryProviderImpl{
kubeletCreationTime: kubeletCreationTime,
systemBootTime: metav1.NewTime(bootTime),
provider: statsProvider,
}
}

func (sp *summaryProviderImpl) Get(updateStats bool) (*statsapi.Summary, error) {
@@ -77,7 +95,7 @@ func (sp *summaryProviderImpl) Get(updateStats bool) (*statsapi.Summary, error)
CPU: rootStats.CPU,
Memory: rootStats.Memory,
Network: networkStats,
StartTime: rootStats.StartTime,
StartTime: sp.systemBootTime,
Fs: rootFsStats,
Runtime: &statsapi.RuntimeStats{ImageFs: imageFsStats},
Rlimit: rlimit,
@@ -86,11 +104,12 @@ func (sp *summaryProviderImpl) Get(updateStats bool) (*statsapi.Summary, error)
systemContainers := map[string]struct {
name string
forceStatsUpdate bool
startTime metav1.Time
}{
statsapi.SystemContainerKubelet: {nodeConfig.KubeletCgroupsName, false},
statsapi.SystemContainerRuntime: {nodeConfig.RuntimeCgroupsName, false},
statsapi.SystemContainerMisc: {nodeConfig.SystemCgroupsName, false},
statsapi.SystemContainerPods: {sp.provider.GetPodCgroupRoot(), updateStats},
statsapi.SystemContainerKubelet: {name: nodeConfig.KubeletCgroupsName, forceStatsUpdate: false, startTime: sp.kubeletCreationTime},
statsapi.SystemContainerRuntime: {name: nodeConfig.RuntimeCgroupsName, forceStatsUpdate: false},
statsapi.SystemContainerMisc: {name: nodeConfig.SystemCgroupsName, forceStatsUpdate: false},
statsapi.SystemContainerPods: {name: sp.provider.GetPodCgroupRoot(), forceStatsUpdate: updateStats},
}
for sys, cont := range systemContainers {
// skip if cgroup name is undefined (not all system containers are required)
@@ -105,6 +124,11 @@ func (sp *summaryProviderImpl) Get(updateStats bool) (*statsapi.Summary, error)
// System containers don't have a filesystem associated with them.
s.Logs, s.Rootfs = nil, nil
s.Name = sys
// if we know the start time of a system container, use that instead of the start time provided by cAdvisor
if !cont.startTime.IsZero() {
s.StartTime = cont.startTime
}

nodeStats.SystemContainers = append(nodeStats.SystemContainers, *s)
}

@@ -80,12 +80,14 @@ func TestSummaryProvider(t *testing.T) {
On("GetCgroupStats", "/kubelet", false).Return(cgroupStatsMap["/kubelet"].cs, cgroupStatsMap["/kubelet"].ns, nil).
On("GetCgroupStats", "/kubepods", true).Return(cgroupStatsMap["/pods"].cs, cgroupStatsMap["/pods"].ns, nil)

provider := NewSummaryProvider(mockStatsProvider)
kubeletCreationTime := metav1.Now()
systemBootTime := metav1.Now()
provider := summaryProviderImpl{kubeletCreationTime: kubeletCreationTime, systemBootTime: systemBootTime, provider: mockStatsProvider}
summary, err := provider.Get(true)
assert.NoError(err)

assert.Equal(summary.Node.NodeName, "test-node")
assert.Equal(summary.Node.StartTime, cgroupStatsMap["/"].cs.StartTime)
assert.Equal(summary.Node.StartTime, systemBootTime)
assert.Equal(summary.Node.CPU, cgroupStatsMap["/"].cs.CPU)
assert.Equal(summary.Node.Memory, cgroupStatsMap["/"].cs.Memory)
assert.Equal(summary.Node.Network, cgroupStatsMap["/"].ns)
@@ -95,7 +97,7 @@ func TestSummaryProvider(t *testing.T) {
assert.Equal(len(summary.Node.SystemContainers), 4)
assert.Contains(summary.Node.SystemContainers, statsapi.ContainerStats{
Name: "kubelet",
StartTime: cgroupStatsMap["/kubelet"].cs.StartTime,
StartTime: kubeletCreationTime,
CPU: cgroupStatsMap["/kubelet"].cs.CPU,
Memory: cgroupStatsMap["/kubelet"].cs.Memory,
Accelerators: cgroupStatsMap["/kubelet"].cs.Accelerators,
@@ -27,6 +27,7 @@ go_library(
"util_unsupported.go",
],
"@io_bazel_rules_go//go/platform:darwin": [
"boottime_util_darwin.go",
"util_unix.go",
],
"@io_bazel_rules_go//go/platform:dragonfly": [
@@ -36,6 +37,7 @@ go_library(
"util_unix.go",
],
"@io_bazel_rules_go//go/platform:linux": [
"boottime_util_linux.go",
"util_unix.go",
],
"@io_bazel_rules_go//go/platform:nacl": [
@@ -0,0 +1,44 @@
// +build darwin

/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
"fmt"
"syscall"
"time"
"unsafe"

"golang.org/x/sys/unix"
)

// GetBootTime returns the time at which the machine was started, truncated to the nearest second
func GetBootTime() (time.Time, error) {
output, err := unix.SysctlRaw("kern.boottime")
if err != nil {
return time.Time{}, err
}
var timeval syscall.Timeval
if len(output) != int(unsafe.Sizeof(timeval)) {
return time.Time{}, fmt.Errorf("unexpected output when calling syscall kern.bootime. Expected len(output) to be %v, but got %v",
int(unsafe.Sizeof(timeval)), len(output))
}
timeval = *(*syscall.Timeval)(unsafe.Pointer(&output[0]))
sec, nsec := timeval.Unix()
return time.Unix(sec, nsec).Truncate(time.Second), nil
}
@@ -0,0 +1,36 @@
// +build freebsd linux

/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
"fmt"
"time"

"golang.org/x/sys/unix"
)

// GetBootTime returns the time at which the machine was started, truncated to the nearest second
func GetBootTime() (time.Time, error) {
currentTime := time.Now()
var info unix.Sysinfo_t
if err := unix.Sysinfo(&info); err != nil {
return time.Time{}, fmt.Errorf("error getting system uptime: %s", err)
}
return currentTime.Add(-time.Duration(info.Uptime) * time.Second).Truncate(time.Second), nil
}
@@ -40,3 +40,8 @@ func LockAndCheckSubPath(volumePath, subPath string) ([]uintptr, error) {
// UnlockPath empty implementation
func UnlockPath(fileHandles []uintptr) {
}

// GetBootTime empty implementation
func GetBootTime() (time.Time, error) {
return time.Time{}, fmt.Errorf("GetBootTime is unsupported in this build")
}
@@ -21,6 +21,7 @@ package util
import (
"fmt"
"net"
"syscall"
"time"
)

@@ -55,3 +56,15 @@ func GetAddressAndDialer(endpoint string) (string, func(addr string, timeout tim
func dial(addr string, timeout time.Duration) (net.Conn, error) {
return net.DialTimeout(tcpProtocol, addr, timeout)
}

var tickCount = syscall.NewLazyDLL("kernel32.dll").NewProc("GetTickCount64")

// GetBootTime returns the time at which the machine was started, truncated to the nearest second
func GetBootTime() (time.Time, error) {
currentTime := time.Now()
output, _, err := tickCount.Call()
if errno, ok := err.(syscall.Errno); !ok || errno != 0 {
return time.Time{}, err
}
return currentTime.Add(-time.Duration(output) * time.Millisecond).Truncate(time.Second), nil
}
@@ -60,6 +60,7 @@ go_library(
] + select({
"@io_bazel_rules_go//go/platform:linux": [
"//pkg/api/v1/node:go_default_library",
"//pkg/kubelet/util:go_default_library",
"//pkg/util/procfs:go_default_library",
"//test/e2e/perftype:go_default_library",
"//test/e2e_node/perftype:go_default_library",
@@ -22,7 +22,6 @@ import (
"fmt"
"os"
"path"
"syscall"
"time"

"k8s.io/api/core/v1"
@@ -34,6 +33,7 @@ import (
clientset "k8s.io/client-go/kubernetes"
coreclientset "k8s.io/client-go/kubernetes/typed/core/v1"
nodeutil "k8s.io/kubernetes/pkg/api/v1/node"
"k8s.io/kubernetes/pkg/kubelet/util"
"k8s.io/kubernetes/test/e2e/framework"

. "github.com/onsi/ginkgo"
@@ -97,8 +97,11 @@ var _ = framework.KubeDescribe("NodeProblemDetector [NodeFeature:NodeProblemDete
BeforeEach(func() {
By("Calculate Lookback duration")
var err error
nodeTime, bootTime, err = getNodeTime()

nodeTime = time.Now()
bootTime, err = util.GetBootTime()
Expect(err).To(BeNil())

// Set lookback duration longer than node up time.
// Assume the test won't take more than 1 hour, in fact it usually only takes 90 seconds.
lookback = nodeTime.Sub(bootTime) + time.Hour
@@ -387,24 +390,6 @@ func injectLog(file string, timestamp time.Time, log string, num int) error {
return nil
}

// getNodeTime gets node boot time and current time.
func getNodeTime() (time.Time, time.Time, error) {
// Get node current time.
nodeTime := time.Now()

// Get system uptime.
var info syscall.Sysinfo_t
if err := syscall.Sysinfo(&info); err != nil {
return time.Time{}, time.Time{}, err
}
// Get node boot time. NOTE that because we get node current time before uptime, the boot time
// calculated will be a little earlier than the real boot time. This won't affect the correctness
// of the test result.
bootTime := nodeTime.Add(-time.Duration(info.Uptime) * time.Second)

return nodeTime, bootTime, nil
}

// verifyEvents verifies there are num specific events generated
func verifyEvents(e coreclientset.EventInterface, options metav1.ListOptions, num int, reason, message string) error {
events, err := e.List(options)

0 comments on commit ede55fd

Please sign in to comment.
You can’t perform that action at this time.