This repository has been archived by the owner on Jan 11, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 561
/
runner.go
242 lines (225 loc) · 6.96 KB
/
runner.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
package main
import (
"fmt"
"log"
"os"
"os/signal"
"path/filepath"
"time"
"github.com/Azure/acs-engine/test/e2e/azure"
"github.com/Azure/acs-engine/test/e2e/config"
"github.com/Azure/acs-engine/test/e2e/engine"
"github.com/Azure/acs-engine/test/e2e/metrics"
outil "github.com/Azure/acs-engine/test/e2e/openshift/util"
"github.com/Azure/acs-engine/test/e2e/runner"
)
var (
cfg *config.Config
acct *azure.Account
eng *engine.Engine
rgs []string
err error
pt *metrics.Point
cliProvisioner *runner.CLIProvisioner
)
func main() {
cwd, _ := os.Getwd()
cfg, err = config.ParseConfig()
if err != nil {
log.Fatalf("Error while trying to parse configuration: %s\n", err)
}
cfg.CurrentWorkingDir = cwd
acct, err = azure.NewAccount()
if err != nil {
log.Fatalf("Error while trying to setup azure account: %s\n", err)
}
err := acct.Login()
if err != nil {
log.Fatalf("Error while trying to login to azure account! %s\n", err)
}
err = acct.SetSubscription()
if err != nil {
log.Fatal("Error while trying to set azure subscription!")
}
pt = metrics.BuildPoint(cfg.Orchestrator, cfg.Location, cfg.ClusterDefinition, acct.SubscriptionID)
// If an interrupt/kill signal is sent we will run the clean up procedure
trap()
cliProvisioner, err = runner.BuildCLIProvisioner(cfg, acct, pt)
if err != nil {
log.Fatalf("Error while trying to build CLI Provisioner:%s", err)
}
sa := acct.StorageAccount
// Soak test specific setup
if cfg.SoakClusterName != "" {
sa.Name = "acsesoaktests" + cfg.Location
sa.ResourceGroup.Name = "acse-test-infrastructure-storage"
sa.ResourceGroup.Location = cfg.Location
err = sa.CreateStorageAccount()
if err != nil {
log.Fatalf("Error while trying to create storage account: %s\n", err)
}
err = sa.SetConnectionString()
if err != nil {
log.Fatalf("Error while trying to set storage account connection string: %s\n", err)
}
provision := true
rg := cfg.SoakClusterName
err = acct.SetResourceGroup(rg)
if err != nil {
log.Printf("Error while trying to set RG:%s\n", err)
} else {
// set expiration time to 7 days = 168h for now
d, err := time.ParseDuration("168h")
if err != nil {
log.Fatalf("Unexpected error parsing duration: %s", err)
}
provision = acct.IsClusterExpired(d)
}
if provision || cfg.ForceDeploy {
log.Printf("Soak cluster %s does not exist or has expired\n", rg)
log.Printf("Deleting Resource Group:%s\n", rg)
acct.DeleteGroup(rg, true)
log.Printf("Deleting Storage files:%s\n", rg)
sa.DeleteFiles(cfg.SoakClusterName)
cfg.Name = ""
} else {
log.Printf("Soak cluster %s exists, downloading output files from storage...\n", rg)
err = sa.DownloadFiles(cfg.SoakClusterName, "_output")
if err != nil {
log.Printf("Error while trying to download _output dir: %s, will provision a new cluster.\n", err)
log.Printf("Deleting Resource Group:%s\n", rg)
acct.DeleteGroup(rg, true)
log.Printf("Deleting Storage files:%s\n", rg)
sa.DeleteFiles(cfg.SoakClusterName)
cfg.Name = ""
} else {
cfg.SetSSHKeyPermissions()
}
}
}
// Only provision a cluster if there isn't a name present
if cfg.Name == "" {
err = cliProvisioner.Run()
rgs = cliProvisioner.ResourceGroups
eng = cliProvisioner.Engine
if err != nil {
if cfg.CleanUpIfFail {
teardown()
}
log.Fatalf("Error while trying to provision cluster:%s", err)
}
if cfg.SoakClusterName != "" {
err = sa.CreateFileShare(cfg.SoakClusterName)
if err != nil {
log.Printf("Error while trying to create file share:%s\n", err)
}
err = sa.UploadFiles(filepath.Join(cfg.CurrentWorkingDir, "_output"), cfg.SoakClusterName)
if err != nil {
log.Fatalf("Error while trying to upload _output dir:%s\n", err)
}
}
} else {
cliProvisioner.ResourceGroups = append(rgs, cliProvisioner.Config.Name)
engCfg, err := engine.ParseConfig(cfg.CurrentWorkingDir, cfg.ClusterDefinition, cfg.Name)
cfg.SetKubeConfig()
if err != nil {
teardown()
log.Fatalf("Error trying to parse Engine config:%s\n", err)
}
cs, err := engine.ParseInput(engCfg.ClusterDefinitionTemplate)
if err != nil {
teardown()
log.Fatalf("Error trying to parse engine template into memory:%s\n", err)
}
eng = &engine.Engine{
Config: engCfg,
ClusterDefinition: cs,
}
cliProvisioner.Engine = eng
}
if !cfg.SkipTest {
g, err := runner.BuildGinkgoRunner(cfg, pt)
if err != nil {
teardown()
log.Fatalf("Error: Unable to parse ginkgo configuration!")
}
err = g.Run()
if err != nil {
teardown()
os.Exit(1)
}
}
teardown()
os.Exit(0)
}
func trap() {
// If an interrupt/kill signal is sent we will run the clean up procedure
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt)
signal.Notify(c, os.Kill)
go func() {
for sig := range c {
log.Printf("Received Signal:%s ... Clean Up On Exit?:%v\n", sig.String(), cfg.CleanUpOnExit)
teardown()
os.Exit(1)
}
}()
}
func teardown() {
pt.RecordTotalTime()
pt.Write()
hostname := fmt.Sprintf("%s.%s.cloudapp.azure.com", cfg.Name, cfg.Location)
logsPath := filepath.Join(cfg.CurrentWorkingDir, "_logs", hostname)
err := os.MkdirAll(logsPath, 0755)
if err != nil {
log.Printf("cannot create directory for logs: %s", err)
}
if cliProvisioner.Config.IsKubernetes() && cfg.SoakClusterName == "" && !cfg.SkipLogsCollection {
err = cliProvisioner.FetchProvisioningMetrics(logsPath, cfg, acct)
if err != nil {
log.Printf("cliProvisioner.FetchProvisioningMetrics error: %s\n", err)
}
}
if cliProvisioner.Config.IsOpenShift() {
sshKeyPath := cfg.GetSSHKeyPath()
adminName := eng.ClusterDefinition.Properties.LinuxProfile.AdminUsername
version := eng.Config.OrchestratorVersion
distro := eng.Config.Distro
if err := outil.FetchWaagentLogs(sshKeyPath, adminName, cfg.Name, cfg.Location, logsPath); err != nil {
log.Printf("cannot fetch waagent logs: %v", err)
}
if err := outil.FetchOpenShiftLogs(distro, version, sshKeyPath, adminName, cfg.Name, cfg.Location, logsPath); err != nil {
log.Printf("cannot get openshift logs: %v", err)
}
if err := outil.FetchClusterInfo(logsPath); err != nil {
log.Printf("cannot get pod and node info: %v", err)
}
if err := outil.FetchOpenShiftMetrics(logsPath); err != nil {
log.Printf("cannot fetch openshift metrics: %v", err)
}
}
if !cfg.SkipLogsCollection {
if err := cliProvisioner.FetchActivityLog(acct, logsPath); err != nil {
log.Printf("cannot fetch the activity log: %v", err)
}
}
if !cfg.RetainSSH {
creds := filepath.Join(cfg.CurrentWorkingDir, "_output/", "*ssh*")
files, err := filepath.Glob(creds)
if err != nil {
log.Printf("failed to get ssh files using %s: %s\n", creds, err)
}
for _, file := range files {
err := os.Remove(file)
if err != nil {
log.Printf("failed to delete file %s: %s\n", file, err)
}
}
}
if cfg.CleanUpOnExit {
for _, rg := range rgs {
log.Printf("Deleting Group:%s\n", rg)
acct.DeleteGroup(rg, false)
}
}
}