Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

kubelet: only check for container listed in the manifest #1670

Merged
merged 4 commits into from
Oct 9, 2014
Merged

kubelet: only check for container listed in the manifest #1670

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
14c5fd4
kubelet: only check for container listed in the manifest
proppy Oct 8, 2014
dbb2342
runonce: list kubelet container between each iteration
proppy Oct 9, 2014
8f010c3
dockertools: add todo for container lookup
proppy Oct 9, 2014
0fc5e68
runonce: isPodRunning returns no error
proppy Oct 9, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions pkg/kubelet/dockertools/docker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,7 @@ type DockerContainers map[DockerID]*docker.APIContainers

func (c DockerContainers) FindPodContainer(podFullName, uuid, containerName string) (*docker.APIContainers, bool, uint64) {
for _, dockerContainer := range c {
// TODO(proppy): build the docker container name and do a map lookup instead?
dockerManifestID, dockerUUID, dockerContainerName, hash := ParseDockerName(dockerContainer.Names[0])
if dockerManifestID == podFullName &&
(uuid == "" || dockerUUID == uuid) &&
Expand Down
55 changes: 24 additions & 31 deletions pkg/kubelet/runonce.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ import (
"fmt"
"time"

"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
"github.com/GoogleCloudPlatform/kubernetes/pkg/kubelet/dockertools"
"github.com/golang/glog"
)
Expand All @@ -33,9 +32,8 @@ const (
)

type RunPodResult struct {
Pod *Pod
Info api.PodInfo
Err error
Pod *Pod
Err error
}

// RunOnce polls from one configuration update and run the associated pods.
Expand All @@ -62,8 +60,8 @@ func (kl *Kubelet) runOnce(pods []Pod) (results []RunPodResult, err error) {
for i := range pods {
pod := pods[i] // Make a copy
go func() {
info, err := kl.runPod(pod)
ch <- RunPodResult{&pod, info, err}
err := kl.runPod(pod)
ch <- RunPodResult{&pod, err}
}()
}

Expand All @@ -73,10 +71,11 @@ func (kl *Kubelet) runOnce(pods []Pod) (results []RunPodResult, err error) {
res := <-ch
results = append(results, res)
if res.Err != nil {
// TODO(proppy): report which containers failed the pod.
glog.Infof("failed to start pod %q: %v", res.Pod.Name, res.Err)
failedPods = append(failedPods, res.Pod.Name)
} else {
glog.Infof("started pod %q: %#v", res.Pod.Name, res.Info)
glog.Infof("started pod %q", res.Pod.Name)
}
}
if len(failedPods) > 0 {
Expand All @@ -86,45 +85,39 @@ func (kl *Kubelet) runOnce(pods []Pod) (results []RunPodResult, err error) {
return results, err
}

// Run a single pod and wait until all containers are running.
func (kl *Kubelet) runPod(pod Pod) (api.PodInfo, error) {
dockerContainers, err := dockertools.GetKubeletDockerContainers(kl.dockerClient, false)
if err != nil {
return nil, fmt.Errorf("failed to get kubelet docker containers: %v", err)
}

// runPod runs a single pod and wait until all containers are running.
func (kl *Kubelet) runPod(pod Pod) error {
delay := RunOnceRetryDelay
retry := 0
for {
glog.Infof("syncing pod")
err := kl.syncPod(&pod, dockerContainers)
dockerContainers, err := dockertools.GetKubeletDockerContainers(kl.dockerClient, false)
if err != nil {
return nil, fmt.Errorf("error syncing pod: %v", err)
return fmt.Errorf("failed to get kubelet docker containers: %v", err)
}
info, err := kl.GetPodInfo(GetPodFullName(&pod), pod.Manifest.UUID)
if err != nil {
return nil, fmt.Errorf("error getting pod info: %v", err)
if running := kl.isPodRunning(pod, dockerContainers); running {
glog.Infof("pod %q containers running", pod.Name)
return nil
}
if podInfo(info).isRunning() {
return info, nil
glog.Infof("pod %q containers not running: syncing", pod.Name)
if err = kl.syncPod(&pod, dockerContainers); err != nil {
return fmt.Errorf("error syncing pod: %v", err)
}
if retry >= RunOnceMaxRetries {
return nil, fmt.Errorf("timeout error: pod %q containers not running after %d retries", pod.Name, RunOnceMaxRetries)
return fmt.Errorf("timeout error: pod %q containers not running after %d retries", pod.Name, RunOnceMaxRetries)
}
glog.Infof("pod %q containers not running, waiting for %v", pod.Name, delay)
// TODO(proppy): health checking would be better than waiting + checking the state at the next iteration.
glog.Infof("pod %q containers synced, waiting for %v", pod.Name, delay)
<-time.After(delay)
retry++
delay *= RunOnceRetryDelayBackoff
}
}

// Alias PodInfo for internal usage.
type podInfo api.PodInfo

// Check if all containers of a pod are running.
func (info podInfo) isRunning() bool {
for _, container := range info {
if container.State.Running == nil {
// isPodRunning returns true if all containers of a manifest are running.
func (kl *Kubelet) isPodRunning(pod Pod, dockerContainers dockertools.DockerContainers) bool {
for _, container := range pod.Manifest.Containers {
if dockerContainer, found, _ := dockerContainers.FindPodContainer(GetPodFullName(&pod), pod.Manifest.UUID, container.Name); !found || dockerContainer.Status != "running" {
glog.Infof("container %q not found (%v) or not running: %#v", container.Name, found, dockerContainer)
return false
}
}
Expand Down
31 changes: 17 additions & 14 deletions pkg/kubelet/runonce_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,22 +67,25 @@ func (d *testDocker) InspectContainer(id string) (*docker.Container, error) {

func TestRunOnce(t *testing.T) {
kb := &Kubelet{}
container := api.Container{Name: "bar"}
podContainers := []docker.APIContainers{
{
Names: []string{"/k8s_bar." + strconv.FormatUint(dockertools.HashContainer(&api.Container{Name: "bar"}), 16) + "_foo.test"},
ID: "1234",
Status: "running",
},
{
Names: []string{"/k8s_net_foo.test_"},
ID: "9876",
Status: "running",
},
}
kb.dockerClient = &testDocker{
listContainersResults: []listContainersResult{
{label: "pre syncPod", containers: []docker.APIContainers{}},
{label: "syncPod #1", containers: []docker.APIContainers{}},
{label: "syncPod #2", containers: []docker.APIContainers{}},
{label: "post syncPod", containers: []docker.APIContainers{
{
Names: []string{"/k8s_bar." + strconv.FormatUint(dockertools.HashContainer(&container), 16) + "_foo.test"},
ID: "1234",
},
{
Names: []string{"/k8s_net_foo.test_"},
ID: "9876",
},
}},
{label: "list pod container", containers: []docker.APIContainers{}},
{label: "syncPod", containers: []docker.APIContainers{}},
{label: "list pod container", containers: []docker.APIContainers{}},
{label: "syncPod", containers: podContainers},
{label: "list pod container", containers: podContainers},
},
inspectContainersResults: []inspectContainersResult{
{
Expand Down