From b91133f913f769e29f787da6ea9333569815acc6 Mon Sep 17 00:00:00 2001 From: Aleksey Pesternikov Date: Sat, 13 Apr 2024 23:22:46 -0700 Subject: [PATCH 1/3] BackOff is a reason to early termination --- testing/it_sidecar/it_sidecar.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testing/it_sidecar/it_sidecar.go b/testing/it_sidecar/it_sidecar.go index e5081d40..d6c5858d 100644 --- a/testing/it_sidecar/it_sidecar.go +++ b/testing/it_sidecar/it_sidecar.go @@ -91,6 +91,7 @@ func contains(v []string, item string) bool { } return false } + // listReadyApps converts a list returned from podsInformer.GetStore().List() to a map containing apps with ready status // app is determined by app label func listReadyApps(list []interface{}) (readypods, notReady []string) { @@ -138,7 +139,7 @@ func listenForEvents(ctx context.Context, clientset *kubernetes.Clientset, onFai return } log.Printf("EVENT %s %s %s %s\n", event.Namespace, event.InvolvedObject.Name, event.Reason, event.Message) - if event.Reason == "Failed" { + if event.Reason == "Failed" || event.Reason == "BackOff" { onFailure(event) } } From d69b2bf0e0e8bc0a5e6fe3bfd0d23d1a53cdd6ee Mon Sep 17 00:00:00 2001 From: Aleksey Pesternikov Date: Sun, 14 Apr 2024 08:55:15 -0700 Subject: [PATCH 2/3] terminate early if container exited with error --- testing/it_sidecar/it_sidecar.go | 2 +- testing/it_sidecar/stern/main.go | 6 +++--- testing/it_sidecar/stern/watch.go | 8 +++++++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/testing/it_sidecar/it_sidecar.go b/testing/it_sidecar/it_sidecar.go index d6c5858d..84c128e5 100644 --- a/testing/it_sidecar/it_sidecar.go +++ b/testing/it_sidecar/it_sidecar.go @@ -371,7 +371,7 @@ func main() { clientset = kubernetes.NewForConfigOrDie(config) defer cleanup(clientset) - go stern.Run(ctx, *namespace, clientset) + go stern.Run(ctx, *namespace, clientset, allowErrors) listenForEvents(ctx, clientset, func(event *v1.Event) { if !allowErrors { diff --git a/testing/it_sidecar/stern/main.go b/testing/it_sidecar/stern/main.go index 5bc5d6b6..0856c774 100644 --- a/testing/it_sidecar/stern/main.go +++ b/testing/it_sidecar/stern/main.go @@ -23,11 +23,11 @@ import ( ) // Run starts the main run loop -func Run(ctx context.Context, namespace string, clientset *kubernetes.Clientset) error { +func Run(ctx context.Context, namespace string, clientset *kubernetes.Clientset, allowErrors bool) error { tails := make(map[string]*Tail) - err := Watch(ctx, clientset.CoreV1().Pods(namespace), RUNNING, labels.Everything(), func(p *Target) { + err := Watch(ctx, clientset.CoreV1().Pods(namespace), RUNNING, labels.Everything(), allowErrors, func(p *Target) { id := p.GetID() if tails[id] != nil { return @@ -45,7 +45,7 @@ func Run(ctx context.Context, namespace string, clientset *kubernetes.Clientset) delete(tails, id) }) if err != nil { - return fmt.Errorf("failed to set up watch: %v", err) + return fmt.Errorf("watch error: %v", err) } return nil diff --git a/testing/it_sidecar/stern/watch.go b/testing/it_sidecar/stern/watch.go index b52328da..e4699fcb 100644 --- a/testing/it_sidecar/stern/watch.go +++ b/testing/it_sidecar/stern/watch.go @@ -41,7 +41,7 @@ func (t *Target) GetID() string { // Watch starts listening to Kubernetes events and emits modified // containers/pods. The first result is targets added, the second is targets // removed -func Watch(ctx context.Context, i v1.PodInterface, containerState ContainerState, labelSelector labels.Selector, onAdded, onRemoved func(*Target)) error { +func Watch(ctx context.Context, i v1.PodInterface, containerState ContainerState, labelSelector labels.Selector, allowErrors bool, onAdded, onRemoved func(*Target)) error { watcher, err := i.Watch(ctx, metav1.ListOptions{Watch: true, LabelSelector: labelSelector.String()}) if err != nil { return fmt.Errorf("failed to set up watch: %s", err) @@ -83,6 +83,12 @@ func Watch(ctx context.Context, i v1.PodInterface, containerState ContainerState log.Print("container ", c.Name, " has state ", c.State) + if !allowErrors { + if t := c.State.Terminated; t != nil && t.ExitCode != 0 && t.Reason == "Error" { + return fmt.Errorf("container %s failed with exit code %d and reason '%s'", c.Name, t.ExitCode, t.Reason) + } + } + if containerState.Match(c.State) { onAdded(&Target{ Namespace: pod.Namespace, From 371d8ee482d031e7e77cc92925a41319c962a8aa Mon Sep 17 00:00:00 2001 From: Aleksey Pesternikov Date: Sun, 14 Apr 2024 09:12:20 -0700 Subject: [PATCH 3/3] cancel the context and finish if stern exited --- testing/it_sidecar/it_sidecar.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/testing/it_sidecar/it_sidecar.go b/testing/it_sidecar/it_sidecar.go index 84c128e5..60a69bb4 100644 --- a/testing/it_sidecar/it_sidecar.go +++ b/testing/it_sidecar/it_sidecar.go @@ -371,7 +371,13 @@ func main() { clientset = kubernetes.NewForConfigOrDie(config) defer cleanup(clientset) - go stern.Run(ctx, *namespace, clientset, allowErrors) + go func() { + err := stern.Run(ctx, *namespace, clientset, allowErrors) + if err != nil { + log.Print(err) + } + cancel() + }() listenForEvents(ctx, clientset, func(event *v1.Event) { if !allowErrors {