Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Automated cherry pick of #104287: Reduce calls to docker from dockershim for stats #106504

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
12 changes: 2 additions & 10 deletions pkg/kubelet/dockershim/docker_image_linux.go
Expand Up @@ -24,20 +24,12 @@ import (
"path/filepath"
"time"

"k8s.io/klog/v2"

runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
)

// ImageFsInfo returns information of the filesystem that is used to store images.
func (ds *dockerService) ImageFsInfo(_ context.Context, _ *runtimeapi.ImageFsInfoRequest) (*runtimeapi.ImageFsInfoResponse, error) {
info, err := ds.client.Info()
if err != nil {
klog.Errorf("Failed to get docker info: %v", err)
return nil, err
}

bytes, inodes, err := dirSize(filepath.Join(info.DockerRootDir, "image"))
bytes, inodes, err := dirSize(filepath.Join(ds.dockerRootDir, "image"))
if err != nil {
return nil, err
}
Expand All @@ -47,7 +39,7 @@ func (ds *dockerService) ImageFsInfo(_ context.Context, _ *runtimeapi.ImageFsInf
{
Timestamp: time.Now().Unix(),
FsId: &runtimeapi.FilesystemIdentifier{
Mountpoint: info.DockerRootDir,
Mountpoint: ds.dockerRootDir,
},
UsedBytes: &runtimeapi.UInt64Value{
Value: uint64(bytes),
Expand Down
12 changes: 3 additions & 9 deletions pkg/kubelet/dockershim/docker_image_windows.go
Expand Up @@ -30,16 +30,10 @@ import (

// ImageFsInfo returns information of the filesystem that is used to store images.
func (ds *dockerService) ImageFsInfo(_ context.Context, _ *runtimeapi.ImageFsInfoRequest) (*runtimeapi.ImageFsInfoResponse, error) {
info, err := ds.client.Info()
if err != nil {
klog.Errorf("Failed to get docker info: %v", err)
return nil, err
}

statsClient := &winstats.StatsClient{}
fsinfo, err := statsClient.GetDirFsInfo(info.DockerRootDir)
fsinfo, err := statsClient.GetDirFsInfo(ds.dockerRootDir)
if err != nil {
klog.Errorf("Failed to get dir fsInfo for %q: %v", info.DockerRootDir, err)
klog.Errorf("Failed to get dir fsInfo for %q: %v", ds.dockerRootDir, err)
return nil, err
}

Expand All @@ -48,7 +42,7 @@ func (ds *dockerService) ImageFsInfo(_ context.Context, _ *runtimeapi.ImageFsInf
Timestamp: time.Now().UnixNano(),
UsedBytes: &runtimeapi.UInt64Value{Value: fsinfo.Usage},
FsId: &runtimeapi.FilesystemIdentifier{
Mountpoint: info.DockerRootDir,
Mountpoint: ds.dockerRootDir,
},
},
}
Expand Down
18 changes: 11 additions & 7 deletions pkg/kubelet/dockershim/docker_service.go
Expand Up @@ -255,16 +255,17 @@ func NewDockerService(config *ClientConfig, podSandboxImage string, streamingCon
ds.network = network.NewPluginManager(plug)
klog.Infof("Docker cri networking managed by %v", plug.Name())

dockerInfo, err := ds.client.Info()
if err != nil {
return nil, fmt.Errorf("Failed to execute Info() call to the Docker client")
}
klog.InfoS("Docker Info", "dockerInfo", dockerInfo)
ds.dockerRootDir = dockerInfo.DockerRootDir

// skipping cgroup driver checks for Windows
if runtime.GOOS == "linux" {
// NOTE: cgroup driver is only detectable in docker 1.11+
cgroupDriver := defaultCgroupDriver
dockerInfo, err := ds.client.Info()
klog.Infof("Docker Info: %+v", dockerInfo)
if err != nil {
klog.Errorf("Failed to execute Info() call to the Docker client: %v", err)
klog.Warningf("Falling back to use the default driver: %q", cgroupDriver)
} else if len(dockerInfo.CgroupDriver) == 0 {
if len(dockerInfo.CgroupDriver) == 0 {
klog.Warningf("No cgroup driver is set in Docker")
klog.Warningf("Falling back to use the default driver: %q", cgroupDriver)
} else {
Expand Down Expand Up @@ -312,6 +313,9 @@ type dockerService struct {
// the docker daemon every time we need to do such checks.
versionCache *cache.ObjectCache

// docker root directory
dockerRootDir string

// containerCleanupInfos maps container IDs to the `containerCleanupInfo` structs
// needed to clean up after containers have been removed.
// (see `applyPlatformSpecificDockerConfig` and `performPlatformSpecificContainerCleanup`
Expand Down
1 change: 1 addition & 0 deletions pkg/kubelet/dockershim/docker_service_test.go
Expand Up @@ -90,6 +90,7 @@ func newTestDockerService() (*dockerService, *libdocker.FakeDockerClient, *clock
network: pm,
checkpointManager: ckm,
networkReady: make(map[string]bool),
dockerRootDir: "/docker/root/dir",
}, c, fakeClock
}

Expand Down
17 changes: 14 additions & 3 deletions pkg/kubelet/dockershim/docker_stats.go
Expand Up @@ -20,13 +20,24 @@ package dockershim

import (
"context"
"fmt"

runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
)

// ContainerStats returns stats for a container stats request based on container id.
func (ds *dockerService) ContainerStats(_ context.Context, r *runtimeapi.ContainerStatsRequest) (*runtimeapi.ContainerStatsResponse, error) {
stats, err := ds.getContainerStats(r.ContainerId)
func (ds *dockerService) ContainerStats(ctx context.Context, r *runtimeapi.ContainerStatsRequest) (*runtimeapi.ContainerStatsResponse, error) {
filter := &runtimeapi.ContainerFilter{
Id: r.ContainerId,
}
listResp, err := ds.ListContainers(ctx, &runtimeapi.ListContainersRequest{Filter: filter})
if err != nil {
return nil, err
}
if len(listResp.Containers) != 1 {
return nil, fmt.Errorf("container with id %s not found", r.ContainerId)
}
stats, err := ds.getContainerStats(listResp.Containers[0])
if err != nil {
return nil, err
}
Expand All @@ -51,7 +62,7 @@ func (ds *dockerService) ListContainerStats(ctx context.Context, r *runtimeapi.L

var stats []*runtimeapi.ContainerStats
for _, container := range listResp.Containers {
containerStats, err := ds.getContainerStats(container.Id)
containerStats, err := ds.getContainerStats(container)
if err != nil {
return nil, err
}
Expand Down
28 changes: 8 additions & 20 deletions pkg/kubelet/dockershim/docker_stats_linux.go
Expand Up @@ -19,42 +19,30 @@ limitations under the License.
package dockershim

import (
"context"
"time"

runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
)

func (ds *dockerService) getContainerStats(containerID string) (*runtimeapi.ContainerStats, error) {
info, err := ds.client.Info()
func (ds *dockerService) getContainerStats(c *runtimeapi.Container) (*runtimeapi.ContainerStats, error) {
statsJSON, err := ds.client.GetContainerStats(c.Id)
if err != nil {
return nil, err
}

statsJSON, err := ds.client.GetContainerStats(containerID)
containerJSON, err := ds.client.InspectContainerWithSize(c.Id)
if err != nil {
return nil, err
}

containerJSON, err := ds.client.InspectContainerWithSize(containerID)
if err != nil {
return nil, err
}

statusResp, err := ds.ContainerStatus(context.Background(), &runtimeapi.ContainerStatusRequest{ContainerId: containerID})
if err != nil {
return nil, err
}
status := statusResp.GetStatus()

dockerStats := statsJSON.Stats
timestamp := time.Now().UnixNano()
containerStats := &runtimeapi.ContainerStats{
Attributes: &runtimeapi.ContainerAttributes{
Id: containerID,
Metadata: status.Metadata,
Labels: status.Labels,
Annotations: status.Annotations,
Id: c.Id,
Metadata: c.Metadata,
Labels: c.Labels,
Annotations: c.Annotations,
},
Cpu: &runtimeapi.CpuUsage{
Timestamp: timestamp,
Expand All @@ -66,7 +54,7 @@ func (ds *dockerService) getContainerStats(containerID string) (*runtimeapi.Cont
},
WritableLayer: &runtimeapi.FilesystemUsage{
Timestamp: timestamp,
FsId: &runtimeapi.FilesystemIdentifier{Mountpoint: info.DockerRootDir},
FsId: &runtimeapi.FilesystemIdentifier{Mountpoint: ds.dockerRootDir},
UsedBytes: &runtimeapi.UInt64Value{Value: uint64(*containerJSON.SizeRw)},
},
}
Expand Down
27 changes: 20 additions & 7 deletions pkg/kubelet/dockershim/docker_stats_test.go
Expand Up @@ -22,35 +22,48 @@ import (
"testing"

dockertypes "github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/stretchr/testify/assert"
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
"k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker"
)

func TestContainerStats(t *testing.T) {
labels := map[string]string{containerTypeLabelKey: containerTypeLabelContainer}
tests := map[string]struct {
containerID string
container *libdocker.FakeContainer
containerStats *dockertypes.StatsJSON
calledDetails []libdocker.CalledDetail
}{
"container exists": {
"fake_container",
&libdocker.FakeContainer{ID: "fake_container"},
"k8s_fake_container",
&libdocker.FakeContainer{
ID: "k8s_fake_container",
Name: "k8s_fake_container_1_2_1",
Config: &container.Config{
Labels: labels,
},
},
&dockertypes.StatsJSON{},
[]libdocker.CalledDetail{
libdocker.NewCalledDetail("list", nil),
libdocker.NewCalledDetail("get_container_stats", nil),
libdocker.NewCalledDetail("inspect_container_withsize", nil),
libdocker.NewCalledDetail("inspect_container", nil),
libdocker.NewCalledDetail("inspect_image", nil),
},
},
"container doesn't exists": {
"nonexistant_fake_container",
&libdocker.FakeContainer{ID: "fake_container"},
"k8s_nonexistant_fake_container",
&libdocker.FakeContainer{
ID: "k8s_fake_container",
Name: "k8s_fake_container_1_2_1",
Config: &container.Config{
Labels: labels,
},
},
&dockertypes.StatsJSON{},
[]libdocker.CalledDetail{
libdocker.NewCalledDetail("get_container_stats", nil),
libdocker.NewCalledDetail("list", nil),
},
},
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/kubelet/dockershim/docker_stats_unsupported.go
Expand Up @@ -24,6 +24,6 @@ import (
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
)

func (ds *dockerService) getContainerStats(containerID string) (*runtimeapi.ContainerStats, error) {
func (ds *dockerService) getContainerStats(c *runtimeapi.Container) (*runtimeapi.ContainerStats, error) {
return nil, fmt.Errorf("not implemented")
}
42 changes: 14 additions & 28 deletions pkg/kubelet/dockershim/docker_stats_windows.go
Expand Up @@ -19,7 +19,6 @@ limitations under the License.
package dockershim

import (
"context"
"strings"
"time"

Expand All @@ -28,26 +27,21 @@ import (
"k8s.io/klog/v2"
)

func (ds *dockerService) getContainerStats(containerID string) (*runtimeapi.ContainerStats, error) {
info, err := ds.client.Info()
if err != nil {
return nil, err
}

hcsshimContainer, err := hcsshim.OpenContainer(containerID)
func (ds *dockerService) getContainerStats(c *runtimeapi.Container) (*runtimeapi.ContainerStats, error) {
hcsshimContainer, err := hcsshim.OpenContainer(c.Id)
if err != nil {
// As we moved from using Docker stats to hcsshim directly, we may query HCS with already exited container IDs.
// That will typically happen with init-containers in Exited state. Docker still knows about them but the HCS does not.
// As we don't want to block stats retrieval for other containers, we only log errors.
if !hcsshim.IsNotExist(err) && !hcsshim.IsAlreadyStopped(err) {
klog.V(4).Infof("Error opening container (stats will be missing) '%s': %v", containerID, err)
klog.V(4).Infof("Error opening container (stats will be missing) '%s': %v", c.Id, err)
}
return nil, nil
}
defer func() {
closeErr := hcsshimContainer.Close()
if closeErr != nil {
klog.Errorf("Error closing container '%s': %v", containerID, closeErr)
klog.Errorf("Error closing container '%s': %v", c.Id, closeErr)
}
}()

Expand All @@ -60,30 +54,19 @@ func (ds *dockerService) getContainerStats(containerID string) (*runtimeapi.Cont
// These hcs errors do not have helpers exposed in public package so need to query for the known codes
// https://github.com/microsoft/hcsshim/blob/master/internal/hcs/errors.go
// PR to expose helpers in hcsshim: https://github.com/microsoft/hcsshim/pull/933
klog.V(4).Infof("Container is not in a state that stats can be accessed '%s': %v. This occurs when the container is created but not started.", containerID, err)
klog.V(4).Infof("Container is not in a state that stats can be accessed '%s': %v. This occurs when the container is created but not started.", c.Id, err)
return nil, nil
}
return nil, err
}

containerJSON, err := ds.client.InspectContainerWithSize(containerID)
if err != nil {
return nil, err
}

statusResp, err := ds.ContainerStatus(context.Background(), &runtimeapi.ContainerStatusRequest{ContainerId: containerID})
if err != nil {
return nil, err
}
status := statusResp.GetStatus()

timestamp := time.Now().UnixNano()
containerStats := &runtimeapi.ContainerStats{
Attributes: &runtimeapi.ContainerAttributes{
Id: containerID,
Metadata: status.Metadata,
Labels: status.Labels,
Annotations: status.Annotations,
Id: c.Id,
Metadata: c.Metadata,
Labels: c.Labels,
Annotations: c.Annotations,
},
Cpu: &runtimeapi.CpuUsage{
Timestamp: timestamp,
Expand All @@ -96,8 +79,11 @@ func (ds *dockerService) getContainerStats(containerID string) (*runtimeapi.Cont
},
WritableLayer: &runtimeapi.FilesystemUsage{
Timestamp: timestamp,
FsId: &runtimeapi.FilesystemIdentifier{Mountpoint: info.DockerRootDir},
UsedBytes: &runtimeapi.UInt64Value{Value: uint64(*containerJSON.SizeRw)},
FsId: &runtimeapi.FilesystemIdentifier{Mountpoint: ds.dockerRootDir},
// used bytes from image are not implemented on Windows
// don't query for it since it is expensive to call docker over named pipe
// https://github.com/moby/moby/blob/1ba54a5fd0ba293db3bea46cd67604b593f2048b/daemon/images/image_windows.go#L11-L14
UsedBytes: &runtimeapi.UInt64Value{Value: 0},
},
}
return containerStats, nil
Expand Down
2 changes: 2 additions & 0 deletions pkg/kubelet/dockershim/libdocker/fake_client.go
Expand Up @@ -225,6 +225,7 @@ func convertFakeContainer(f *FakeContainer) *dockertypes.ContainerJSON {
if f.HostConfig == nil {
f.HostConfig = &dockercontainer.HostConfig{}
}
fakeRWSize := int64(40)
return &dockertypes.ContainerJSON{
ContainerJSONBase: &dockertypes.ContainerJSONBase{
ID: f.ID,
Expand All @@ -239,6 +240,7 @@ func convertFakeContainer(f *FakeContainer) *dockertypes.ContainerJSON {
},
Created: dockerTimestampToString(f.CreatedAt),
HostConfig: f.HostConfig,
SizeRw: &fakeRWSize,
},
Config: f.Config,
NetworkSettings: &dockertypes.NetworkSettings{},
Expand Down