-
Notifications
You must be signed in to change notification settings - Fork 9
/
wait.go
109 lines (89 loc) · 3.07 KB
/
wait.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
// Copyright 2022 Namespace Labs Inc; All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
package ctl
import (
"context"
"fmt"
"time"
"github.com/cenkalti/backoff/v4"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/tools/clientcmd"
"namespacelabs.dev/foundation/internal/console"
"namespacelabs.dev/foundation/internal/executor"
"namespacelabs.dev/foundation/internal/fnerrors"
"namespacelabs.dev/foundation/internal/providers/nscloud/api"
"namespacelabs.dev/foundation/internal/runtime/kubernetes/kubeobserver"
"namespacelabs.dev/foundation/std/tasks"
)
const (
kubeSystem = "kube-system"
waitTimeout = 5 * time.Minute
waitBackoff = 500 * time.Millisecond
)
var deployments = []string{
"coredns",
"local-path-provisioner",
}
func WaitKubeSystem(ctx context.Context, cluster *api.KubernetesCluster) error {
return tasks.Action("cluster.wait-kube-system").
Arg("id", cluster.ClusterId).Run(ctx, func(ctx context.Context) error {
cfg := clientcmd.NewDefaultClientConfig(MakeConfig(cluster), nil)
restcfg, err := cfg.ClientConfig()
if err != nil {
return fnerrors.New("failed to load kubernetes configuration: %w", err)
}
eg := executor.New(ctx, "wait")
for _, d := range deployments {
d := d
eg.Go(func(ctx context.Context) error {
fmt.Fprintf(console.Debug(ctx), "will wait for deployment %s\n", d)
obs := kubeobserver.WaitOnResource{
RestConfig: restcfg,
Name: d,
Namespace: kubeSystem,
Description: fmt.Sprintf("kube-system deployment %s", d),
GroupVersionKind: schema.FromAPIVersionAndKind("apps/v1", "Deployment"),
}
return obs.WaitUntilReady(ctx, nil)
})
}
return eg.Wait()
})
}
func WaitContainers(ctx context.Context, clusterId string, ctrs []*api.Container) error {
return tasks.Action("cluster.wait-containers").HumanReadablef("Waiting for containers to start").
Arg("id", clusterId).Run(ctx, func(ctx context.Context) error {
fmt.Fprintf(console.Debug(ctx), "polling cluster %q\n", clusterId)
ctx, cancel := context.WithTimeout(ctx, waitTimeout)
defer cancel()
return backoff.Retry(func() error {
res, err := api.GetClusterSummary(ctx, api.Methods, clusterId, []string{"nsc/containers"})
if err != nil {
return fmt.Errorf("failed to fetch cluster summary: %w", err)
}
resources := map[string]*api.Resource{} // keyed by UID
for _, sum := range res.Summary {
for _, r := range sum.PerResource {
resources[r.Uid] = &r
}
}
for _, ctr := range ctrs {
r, ok := resources[ctr.Id]
if !ok {
return fmt.Errorf("no summary for requested container %q yet", ctr.Id)
}
for _, c := range r.Container {
if !c.Ready {
msg := fmt.Sprintf("container %q is not ready", c.Id)
if c.NotRunningReason != "" {
msg = fmt.Sprintf("%s: %s", msg, c.NotRunningReason)
}
return fmt.Errorf(msg)
}
}
}
return nil
}, backoff.WithContext(backoff.NewConstantBackOff(waitBackoff), ctx))
})
}