Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Cleanup orphan logging that goes on in the sync loop. #44938

Merged
merged 1 commit into from
Apr 28, 2017
Merged

Cleanup orphan logging that goes on in the sync loop. #44938

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
26 changes: 20 additions & 6 deletions pkg/kubelet/kubelet_volumes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ func (kl *Kubelet) newVolumeMounterFromPlugins(spec *volume.Spec, pod *v1.Pod, o
}

// cleanupOrphanedPodDirs removes the volumes of pods that should not be
// running and that have no containers running.
// running and that have no containers running. Note that we roll up logs here since it runs in the main loop.
func (kl *Kubelet) cleanupOrphanedPodDirs(
pods []*v1.Pod, runningPods []*kubecontainer.Pod) error {
allPods := sets.NewString()
Expand All @@ -93,7 +93,10 @@ func (kl *Kubelet) cleanupOrphanedPodDirs(
if err != nil {
return err
}
errlist := []error{}

orphanRemovalErrors := []error{}
orphanVolumeErrors := []error{}

for _, uid := range found {
if allPods.Has(string(uid)) {
continue
Expand All @@ -107,18 +110,29 @@ func (kl *Kubelet) cleanupOrphanedPodDirs(
// If there are still volume directories, do not delete directory
volumePaths, err := kl.getPodVolumePathListFromDisk(uid)
if err != nil {
glog.Errorf("Orphaned pod %q found, but error %v occurred during reading volume dir from disk", uid, err)
orphanVolumeErrors = append(orphanVolumeErrors, fmt.Errorf("Orphaned pod %q found, but error %v occurred during reading volume dir from disk", uid, err))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jayunit100 do you have any insight re: https://bugzilla.redhat.com/show_bug.cgi?id=1445410 -- i find it strange we hit times where err printed nil here ;-)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i think a while back it also used to spam this if there were any volume paths floating around 041fa64#diff-79d1ea3801b52a2add9229183049f4c2

this is fixed now, so it might be lagging in some older forks/distros ., i doubt you will ever see <nil> err running off of the current or any recent (4 months) master fork

continue
}
if len(volumePaths) > 0 {
glog.Errorf("Orphaned pod %q found, but volume paths are still present on disk.", uid)
orphanVolumeErrors = append(orphanVolumeErrors, fmt.Errorf("Orphaned pod %q found, but volume paths are still present on disk.", uid))
continue
}
glog.V(3).Infof("Orphaned pod %q found, removing", uid)
if err := removeall.RemoveAllOneFilesystem(kl.mounter, kl.getPodDir(uid)); err != nil {
glog.Errorf("Failed to remove orphaned pod %q dir; err: %v", uid, err)
errlist = append(errlist, err)
orphanRemovalErrors = append(orphanRemovalErrors, err)
}
}

logSpew := func(errs []error) {
if len(errs) > 0 {
glog.Errorf("%v : There were a total of %v errors similar to this. Turn up verbosity to see them.", errs[0], len(errs))
for _, err := range errs {
glog.V(5).Infof("Orphan pod: %v", err)
}
}
}
return utilerrors.NewAggregate(errlist)
logSpew(orphanVolumeErrors)
logSpew(orphanRemovalErrors)
return utilerrors.NewAggregate(orphanRemovalErrors)
}