/
instances.go
116 lines (98 loc) · 3.33 KB
/
instances.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
package e2e
import (
"fmt"
"log/slog"
"time"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/cloudbase/garm/params"
)
func waitInstanceStatus(name string, status commonParams.InstanceStatus, runnerStatus params.RunnerStatus, timeout time.Duration) (*params.Instance, error) {
var timeWaited time.Duration = 0
var instance *params.Instance
slog.Info("Waiting for instance to reach desired status", "instance", name, "desired_status", status, "desired_runner_status", runnerStatus)
for timeWaited < timeout {
instance, err := getInstance(cli, authToken, name)
if err != nil {
return nil, err
}
slog.Info("Instance status", "instance_name", name, "status", instance.Status, "runner_status", instance.RunnerStatus)
if instance.Status == status && instance.RunnerStatus == runnerStatus {
return instance, nil
}
time.Sleep(5 * time.Second)
timeWaited += 5 * time.Second
}
if err := printJsonResponse(*instance); err != nil {
return nil, err
}
return nil, fmt.Errorf("timeout waiting for instance %s status to reach status %s and runner status %s", name, status, runnerStatus)
}
func DeleteInstance(name string, forceRemove bool) {
if err := deleteInstance(cli, authToken, name, forceRemove); err != nil {
panic(err)
}
slog.Info("Instance deletion initiated", "instance_name", name)
}
func WaitInstanceToBeRemoved(name string, timeout time.Duration) error {
var timeWaited time.Duration = 0
var instance *params.Instance
slog.Info("Waiting for instance to be removed", "instance_name", name)
for timeWaited < timeout {
instances, err := listInstances(cli, authToken)
if err != nil {
return err
}
instance = nil
for _, i := range instances {
if i.Name == name {
instance = &i
break
}
}
if instance == nil {
// The instance is not found in the list. We can safely assume
// that it is removed
return nil
}
time.Sleep(5 * time.Second)
timeWaited += 5 * time.Second
}
if err := printJsonResponse(*instance); err != nil {
return err
}
return fmt.Errorf("instance %s was not removed within the timeout", name)
}
func WaitPoolInstances(poolID string, status commonParams.InstanceStatus, runnerStatus params.RunnerStatus, timeout time.Duration) error {
var timeWaited time.Duration = 0
pool, err := getPool(cli, authToken, poolID)
if err != nil {
return err
}
slog.Info("Waiting for pool instances to reach desired status", "pool_id", poolID, "desired_status", status, "desired_runner_status", runnerStatus)
for timeWaited < timeout {
poolInstances, err := listPoolInstances(cli, authToken, poolID)
if err != nil {
return err
}
instancesCount := 0
for _, instance := range poolInstances {
if instance.Status == status && instance.RunnerStatus == runnerStatus {
instancesCount++
}
}
slog.Info(
"Pool instance reached status",
"pool_id", poolID,
"status", status,
"runner_status", runnerStatus,
"desired_instance_count", instancesCount,
"pool_instance_count", len(poolInstances))
if instancesCount == int(pool.MinIdleRunners) && instancesCount == len(poolInstances) {
return nil
}
time.Sleep(5 * time.Second)
timeWaited += 5 * time.Second
}
_ = dumpPoolInstancesDetails(pool.ID)
return fmt.Errorf("timeout waiting for pool %s instances to reach status: %s and runner status: %s", poolID, status, runnerStatus)
}