Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix podstatus issue caused by docker's resource temporarily unavailable issue #4376

Merged
merged 2 commits into from
Feb 12, 2015
Merged

Fix podstatus issue caused by docker's resource temporarily unavailable issue #4376

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
af54c62
Fix podstatus issue caused by docker's resource temporarily unavailab…
dchen1107 Feb 9, 2015
23b8b5e
Configure docker with larger open files.
dchen1107 Feb 12, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions cluster/saltbase/salt/docker/docker-defaults
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@ DOCKER_OPTS=""
DOCKER_OPTS="${DOCKER_OPTS} {{grains.docker_opts}}"
{% endif %}
DOCKER_OPTS="${DOCKER_OPTS} --bridge cbr0 --iptables=false --ip-masq=false -r=false"
DOCKER_NOFILE=1000000
21 changes: 14 additions & 7 deletions pkg/kubelet/kubelet.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1449,10 +1449,6 @@ func (kl *Kubelet) GetPodByName(namespace, name string) (*api.BoundPod, bool) {

// getPhase returns the phase of a pod given its container info.
func getPhase(spec *api.PodSpec, info api.PodInfo) api.PodPhase {
if info == nil {
return api.PodPending
}

running := 0
waiting := 0
stopped := 0
Expand Down Expand Up @@ -1481,6 +1477,7 @@ func getPhase(spec *api.PodSpec, info api.PodInfo) api.PodPhase {
}
switch {
case waiting > 0:
glog.V(5).Infof("pod waiting > 0, pending")
// One or more containers has not been started
return api.PodPending
case running > 0 && unknown == 0:
Expand All @@ -1507,6 +1504,7 @@ func getPhase(spec *api.PodSpec, info api.PodInfo) api.PodPhase {
// and in the process of restarting
return api.PodRunning
default:
glog.V(5).Infof("pod default case, pending")
return api.PodPending
}
}
Expand Down Expand Up @@ -1555,10 +1553,19 @@ func (kl *Kubelet) GetPodStatus(podFullName string, uid types.UID) (api.PodStatu
info, err := dockertools.GetDockerPodInfo(kl.dockerClient, spec, podFullName, uid)

if err != nil {
glog.Infof("Query docker container info failed with error: %v", err)
return podStatus, err
// Error handling
glog.Infof("Query docker container info for pod %s failed with error (%v)", podFullName, err)
if strings.Contains(err.Error(), "resource temporarily unavailable") {
// Leave upstream layer to decide what to do
return podStatus, err
} else {
podStatus.Phase = api.PodPending
podStatus.Message = fmt.Sprintf("Query docker container info failed with error (%v)", err)
return podStatus, nil
}
}

// Assume info is ready to process
podStatus.Phase = getPhase(&spec, info)
for _, c := range spec.Containers {
containerStatus := info[c.Name]
Expand All @@ -1575,7 +1582,7 @@ func (kl *Kubelet) GetPodStatus(podFullName string, uid types.UID) (api.PodStatu
// TODO(dchen1107): Change Info to list from map
podStatus.Info = info

return podStatus, err
return podStatus, nil
}

// Returns logs of current machine.
Expand Down
10 changes: 5 additions & 5 deletions pkg/master/pod_cache.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,9 @@ func (p *PodCache) updatePodStatus(pod *api.Pod) error {
p.lock.Lock()
defer p.lock.Unlock()
// Map accesses must be locked.
p.podStatus[objKey{pod.Namespace, pod.Name}] = newStatus
if err == nil {
p.podStatus[objKey{pod.Namespace, pod.Name}] = newStatus
}

return err
}
Expand Down Expand Up @@ -187,13 +189,11 @@ func (p *PodCache) computePodStatus(pod *api.Pod) (api.PodStatus, error) {
}

result, err := p.containerInfo.GetPodStatus(pod.Status.Host, pod.Namespace, pod.Name)
newStatus.HostIP = nodeStatus.HostIP

if err != nil {
glog.Errorf("error getting pod status: %v, setting status to unknown", err)
newStatus.Phase = api.PodUnknown
newStatus.Conditions = append(newStatus.Conditions, pod.Status.Conditions...)
glog.Infof("error getting pod %s status: %v, retry later", pod.Name, err)
} else {
newStatus.HostIP = nodeStatus.HostIP
newStatus.Info = result.Status.Info
newStatus.PodIP = result.Status.PodIP
if newStatus.Info == nil {
Expand Down