-
Notifications
You must be signed in to change notification settings - Fork 38.7k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
GC pod ips #35572
GC pod ips #35572
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,117 @@ | ||
/* | ||
Copyright 2015 The Kubernetes Authors. | ||
|
||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
|
||
http://www.apache.org/licenses/LICENSE-2.0 | ||
|
||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
*/ | ||
|
||
package e2e_node | ||
|
||
import ( | ||
"k8s.io/kubernetes/test/e2e/framework" | ||
"time" | ||
|
||
"fmt" | ||
. "github.com/onsi/ginkgo" | ||
"k8s.io/kubernetes/pkg/api" | ||
testutils "k8s.io/kubernetes/test/utils" | ||
"os/exec" | ||
) | ||
|
||
// waitForPods waits for timeout duration, for pod_count. | ||
// If the timeout is hit, it returns the list of currently running pods. | ||
func waitForPods(f *framework.Framework, pod_count int, timeout time.Duration) (runningPods []*api.Pod) { | ||
for start := time.Now(); time.Since(start) < timeout; time.Sleep(10 * time.Second) { | ||
podList, err := f.PodClient().List(api.ListOptions{}) | ||
if err != nil { | ||
framework.Logf("Failed to list pods on node: %v", err) | ||
continue | ||
} | ||
|
||
runningPods = []*api.Pod{} | ||
for _, pod := range podList.Items { | ||
if r, err := testutils.PodRunningReady(&pod); err != nil || !r { | ||
continue | ||
} | ||
runningPods = append(runningPods, &pod) | ||
} | ||
framework.Logf("Running pod count %d", len(runningPods)) | ||
if len(runningPods) >= pod_count { | ||
break | ||
} | ||
} | ||
return runningPods | ||
} | ||
|
||
var _ = framework.KubeDescribe("Restart [Serial] [Slow] [Disruptive]", func() { | ||
const ( | ||
// Saturate the node. It's not necessary that all these pods enter | ||
// Running/Ready, because we don't know the number of cores in the | ||
// test node or default limits applied (if any). It's is essential | ||
// that no containers end up in terminated. 100 was chosen because | ||
// it's the max pods per node. | ||
podCount = 100 | ||
podCreationInterval = 100 * time.Millisecond | ||
recoverTimeout = 5 * time.Minute | ||
startTimeout = 3 * time.Minute | ||
// restartCount is chosen so even with minPods we exhaust the default | ||
// allocation of a /24. | ||
minPods = 50 | ||
restartCount = 6 | ||
) | ||
|
||
f := framework.NewDefaultFramework("restart-test") | ||
Context("Docker Daemon", func() { | ||
Context("Network", func() { | ||
It("should recover from ip leak", func() { | ||
|
||
pods := newTestPods(podCount, framework.GetPauseImageNameForHostArch(), "restart-docker-test") | ||
By(fmt.Sprintf("Trying to create %d pods on node", len(pods))) | ||
createBatchPodWithRateControl(f, pods, podCreationInterval) | ||
defer deletePodsSync(f, pods) | ||
|
||
// Give the node some time to stabilize, assume pods that enter RunningReady within | ||
// startTimeout fit on the node and the node is now saturated. | ||
runningPods := waitForPods(f, podCount, startTimeout) | ||
if len(runningPods) < minPods { | ||
framework.Failf("Failed to start %d pods, cannot test that restarting docker doesn't leak IPs", minPods) | ||
} | ||
|
||
for i := 0; i < restartCount; i += 1 { | ||
By(fmt.Sprintf("Restarting Docker Daemon iteration %d", i)) | ||
|
||
// TODO: Find a uniform way to deal with systemctl/initctl/service operations. #34494 | ||
if stdout, err := exec.Command("sudo", "systemctl", "restart", "docker").CombinedOutput(); err != nil { | ||
framework.Logf("Failed to trigger docker restart with systemd/systemctl: %v, stdout: %q", err, string(stdout)) | ||
if stdout, err = exec.Command("sudo", "service", "docker", "restart").CombinedOutput(); err != nil { | ||
framework.Failf("Failed to trigger docker restart with upstart/service: %v, stdout: %q", err, string(stdout)) | ||
} | ||
} | ||
time.Sleep(20 * time.Second) | ||
} | ||
|
||
By("Checking currently Running/Ready pods") | ||
postRestartRunningPods := waitForPods(f, len(runningPods), recoverTimeout) | ||
if len(postRestartRunningPods) == 0 { | ||
framework.Failf("Failed to start *any* pods after docker restart, this might indicate an IP leak") | ||
} | ||
By("Confirm no containers have terminated") | ||
for _, pod := range postRestartRunningPods { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Shouldn't there always be terminated containers because of the docker restart? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yeah that's lastState, im checking state which shouldn't be terminated at the end of this experiment (it is in step 3 below):
If GC hadn't run, we would be stuck at (3) a 100 containers with state terminated (finishedAt blah), ready=false. |
||
if c := testutils.TerminatedContainers(pod); len(c) != 0 { | ||
framework.Failf("Pod %q has failed containers %+v after docker restart, this might indicate an IP leak", pod.Name, c) | ||
} | ||
} | ||
By(fmt.Sprintf("Docker restart test passed with %d pods", len(postRestartRunningPods))) | ||
}) | ||
}) | ||
}) | ||
}) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
What if the infra container has already terminated? You probably want to check the container state before inserting.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
this is a race we can't easily avoid. If it exits after getNonExitedPods and this line, assume we get a teardown. It's more important that we detect the gargabe in the ip dir.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
My comment wasn't about the race condition.
getNonExitedPods
returns pods with at least one running container, which may include a pod with a running user container and a dead infra container. I don't see the state of the infra container being checked anywhere.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
is that a problem?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Other than the IP used by those infra containers wouldn't be recycled, there is no problem.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
And that pod will get cleaned up the normal way (teardown)? there's no way we restarted an old container because we must've tried the infra container first and failed, so this must be a crashing infra container of a current user pods that will at some point in the future naturally die. no?