Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Automated cherry pick of #98510: Ignore transient errors when gather stats #102499

Merged
merged 1 commit into from Jun 8, 2021
Merged

Automated cherry pick of #98510: Ignore transient errors when gather stats #102499

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
13 changes: 12 additions & 1 deletion pkg/kubelet/dockershim/docker_stats_windows.go
View file
Open in desktop
Expand Up @@ -20,6 +20,7 @@ package dockershim

import (
"context"
"strings"
"time"

"github.com/Microsoft/hcsshim"
Expand All @@ -39,7 +40,7 @@ func (ds *dockerService) getContainerStats(containerID string) (*runtimeapi.Cont
// That will typically happen with init-containers in Exited state. Docker still knows about them but the HCS does not.
// As we don't want to block stats retrieval for other containers, we only log errors.
if !hcsshim.IsNotExist(err) && !hcsshim.IsAlreadyStopped(err) {
klog.Errorf("Error opening container (stats will be missing) '%s': %v", containerID, err)
klog.V(4).Infof("Error opening container (stats will be missing) '%s': %v", containerID, err)
}
return nil, nil
}
Expand All @@ -52,6 +53,16 @@ func (ds *dockerService) getContainerStats(containerID string) (*runtimeapi.Cont

stats, err := hcsshimContainer.Statistics()
if err != nil {
if strings.Contains(err.Error(), "0x5") || strings.Contains(err.Error(), "0xc0370105") {
// When the container is just created, querying for stats causes access errors because it hasn't started yet
// This is transient; skip container for now
//
// These hcs errors do not have helpers exposed in public package so need to query for the known codes
// https://github.com/microsoft/hcsshim/blob/master/internal/hcs/errors.go
// PR to expose helpers in hcsshim: https://github.com/microsoft/hcsshim/pull/933
klog.V(4).Infof("Container is not in a state that stats can be accessed '%s': %v. This occurs when the container is created but not started.", containerID, err)
return nil, nil
}
return nil, err
}

Expand Down