-
Notifications
You must be signed in to change notification settings - Fork 124
/
spawn.go
355 lines (315 loc) · 11.5 KB
/
spawn.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
package cloud
import (
"context"
"encoding/base64"
"fmt"
"net/http"
"strings"
"time"
"github.com/evergreen-ci/birch"
"github.com/evergreen-ci/evergreen"
"github.com/evergreen-ci/evergreen/model"
"github.com/evergreen-ci/evergreen/model/distro"
"github.com/evergreen-ci/evergreen/model/host"
"github.com/evergreen-ci/evergreen/model/user"
"github.com/evergreen-ci/evergreen/util"
"github.com/evergreen-ci/gimlet"
"github.com/evergreen-ci/utility"
"github.com/mongodb/grip"
"github.com/mongodb/grip/message"
"github.com/mongodb/jasper"
"github.com/pkg/errors"
)
// Options holds the required parameters for spawning a host.
type SpawnOptions struct {
DistroId string
Userdata string
UserName string
PublicKey string
ProvisionOptions *host.ProvisionOptions
UseProjectSetupScript bool
InstanceTags []host.Tag
InstanceType string
Region string
NoExpiration bool
IsVirtualWorkstation bool
IsCluster bool
HomeVolumeSize int
HomeVolumeID string
Expiration *time.Time
}
// Validate returns an instance of BadOptionsErr if the SpawnOptions object contains invalid
// data, SpawnLimitErr if the user is already at the spawned host limit, or some other untyped
// instance of Error if something fails during validation.
func (so *SpawnOptions) validate(ctx context.Context, settings *evergreen.Settings) error {
d, err := distro.FindOneId(ctx, so.DistroId)
if err != nil {
return errors.Errorf("finding distro '%s'", so.DistroId)
}
if d == nil {
return errors.Errorf("distro '%s' not found", so.DistroId)
}
if !d.SpawnAllowed {
return errors.Errorf("spawn hosts not allowed for distro '%s'", so.DistroId)
}
// if the user already has too many active spawned hosts, deny the request
activeSpawnedHosts, err := host.Find(ctx, host.ByUserWithRunningStatus(so.UserName))
if err != nil {
return errors.Wrap(err, "finding user's current hosts")
}
if err = checkSpawnHostLimitExceeded(len(activeSpawnedHosts), settings); err != nil {
return err
}
// validate public key
if err = evergreen.ValidateSSHKey(so.PublicKey); err != nil {
return errors.Wrap(err, "invalid SSH key options")
}
sections := strings.Split(so.PublicKey, " ")
if len(sections) < 2 {
return errors.Errorf("missing space in public key")
}
// check for valid base64
if _, err = base64.StdEncoding.DecodeString(sections[1]); err != nil {
return errors.New("public key contains invalid base64 string")
}
return nil
}
func checkSpawnHostLimitExceeded(numCurrentHosts int, settings *evergreen.Settings) error {
if numCurrentHosts >= settings.Spawnhost.SpawnHostsPerUser {
return errors.Errorf("user is already running the max allowed number of spawn hosts (%d of %d)", numCurrentHosts, settings.Spawnhost.SpawnHostsPerUser)
}
return nil
}
// CreateSpawnHost spawns a host with the given options.
func CreateSpawnHost(ctx context.Context, so SpawnOptions, settings *evergreen.Settings) (*host.Host, error) {
if err := so.validate(ctx, settings); err != nil {
return nil, errors.WithStack(err)
}
// load in the appropriate distro
d, err := distro.FindOneId(ctx, so.DistroId)
if err != nil {
return nil, errors.WithStack(errors.Wrap(err, "finding distro"))
}
if d == nil {
return nil, errors.Errorf("distro '%s' not found", so.DistroId)
}
if so.Region == "" && evergreen.IsEc2Provider(d.Provider) {
u := gimlet.GetUser(ctx)
dbUser, ok := u.(*user.DBUser)
if !ok {
return nil, errors.Errorf("getting DBUser from User")
}
so.Region = dbUser.GetRegion()
}
if so.Userdata != "" {
if !evergreen.IsEc2Provider(d.Provider) {
return nil, errors.Errorf("cannot set user data for non-EC2 provider '%s'", d.Provider)
}
if _, err = parseUserData(so.Userdata); err != nil {
return nil, errors.Wrap(err, "user data is malformed")
}
err = d.SetUserdata(so.Userdata, so.Region)
if err != nil {
return nil, errors.WithStack(err)
}
}
if so.HomeVolumeID != "" {
var volume *host.Volume
volume, err = host.ValidateVolumeCanBeAttached(ctx, so.HomeVolumeID)
if err != nil {
return nil, errors.WithStack(err)
}
if AztoRegion(volume.AvailabilityZone) != so.Region {
return nil, errors.Errorf("cannot use volume in zone '%s' with host in region '%s'", volume.AvailabilityZone, so.Region)
}
}
if so.UseProjectSetupScript {
so.ProvisionOptions.SetupScript, err = model.GetSetupScriptForTask(ctx, so.ProvisionOptions.TaskId)
if err != nil {
// still spawn the host if the setup script is buggy
grip.Error(message.WrapError(err, message.Fields{
"message": "failed to get setup script for host",
"task_id": so.ProvisionOptions.TaskId,
"user_id": so.UserName,
}))
}
}
d.ProviderSettingsList, err = modifySpawnHostProviderSettings(*d, settings, so.Region, so.HomeVolumeID)
if err != nil {
return nil, errors.Wrap(err, "getting new provider settings")
}
if so.InstanceType != "" {
if err := CheckInstanceTypeValid(ctx, *d, so.InstanceType, settings.Providers.AWS.AllowedInstanceTypes); err != nil {
return nil, errors.Wrap(err, "validating instance type")
}
}
// modify the setup script to add the user's public key
d.Setup += fmt.Sprintf("\necho \"\n%s\" >> %s\n", so.PublicKey, d.GetAuthorizedKeysFile())
// fake out replacing spot instances with on-demand equivalents
if d.Provider == evergreen.ProviderNameEc2Fleet {
d.Provider = evergreen.ProviderNameEc2OnDemand
}
// spawn the host
currentTime := time.Now()
expiration := evergreen.DefaultSpawnHostExpiration
if so.Expiration != nil {
expiration = so.Expiration.Sub(currentTime)
}
if so.NoExpiration {
expiration = evergreen.SpawnHostNoExpirationDuration
}
hostOptions := host.CreateOptions{
Distro: *d,
ProvisionOptions: so.ProvisionOptions,
UserName: so.UserName,
ExpirationTime: currentTime.Add(expiration),
UserHost: true,
InstanceTags: so.InstanceTags,
InstanceType: so.InstanceType,
NoExpiration: so.NoExpiration,
IsVirtualWorkstation: so.IsVirtualWorkstation,
IsCluster: so.IsCluster,
HomeVolumeSize: so.HomeVolumeSize,
HomeVolumeID: so.HomeVolumeID,
Region: so.Region,
}
intentHost := host.NewIntent(hostOptions)
if intentHost == nil { // theoretically this should not happen
return nil, errors.New("could not create new intent host")
}
return intentHost, nil
}
// assumes distro already modified to have one region
func CheckInstanceTypeValid(ctx context.Context, d distro.Distro, requestedType string, allowedTypes []string) error {
if !utility.StringSliceContains(allowedTypes, requestedType) {
// if it's not in the settings list, check the distro
originalInstanceType, ok := d.ProviderSettingsList[0].Lookup("instance_type").StringValueOK()
if !ok || originalInstanceType != requestedType {
return errors.New("this instance type has not been allowed by admins")
}
}
env := evergreen.GetEnvironment()
opts, err := GetManagerOptions(d)
if err != nil {
return errors.Wrap(err, "getting cloud manager options")
}
m, err := GetManager(ctx, env, opts)
if err != nil {
return errors.Wrap(err, "getting cloud manager")
}
if err := m.CheckInstanceType(ctx, requestedType); err != nil {
return errors.Wrapf(err, "checking instance type '%s'", requestedType)
}
return nil
}
// SetHostRDPPassword is a shared utility function to change the password on a windows host
func SetHostRDPPassword(ctx context.Context, env evergreen.Environment, h *host.Host, pwd string) (int, error) {
if !h.Distro.IsWindows() {
return http.StatusBadRequest, errors.New("RDP password can only be set on Windows hosts")
}
if !host.ValidateRDPPassword(pwd) {
return http.StatusBadRequest, errors.New("invalid password")
}
if h.Status != evergreen.HostRunning {
return http.StatusBadRequest, errors.New("RDP passwords can only be set on running hosts")
}
if err := updateRDPPassword(ctx, env, h, pwd); err != nil {
return http.StatusInternalServerError, err
}
return http.StatusOK, nil
}
func updateRDPPassword(ctx context.Context, env evergreen.Environment, host *host.Host, password string) error {
const redactedPasswordStr = "<REDACTED>"
pwdUpdateCmd, err := constructPwdUpdateCommand(ctx, env, host, password)
if err != nil {
return errors.Wrap(err, "constructing host RDP password")
}
stdout := util.NewMBCappedWriter()
stderr := util.NewMBCappedWriter()
pwdUpdateCmd.SetErrorWriter(stderr).SetOutputWriter(stdout)
// update RDP and sshd password
if err = pwdUpdateCmd.Run(ctx); err != nil {
grip.Warning(message.Fields{
"stdout": stdout.String(),
"stderr": stderr.String(),
"operation": "set host RDP password",
"host_id": host.Id,
"cmd": strings.ReplaceAll(pwdUpdateCmd.String(), password, redactedPasswordStr),
"err": err.Error(),
})
return errors.Wrap(err, "updating host RDP password")
}
grip.Debug(message.Fields{
"stdout": stdout.String(),
"stderr": stderr.String(),
"operation": "set host RDP password",
"host_id": host.Id,
})
return nil
}
// constructPwdUpdateCommand returns a RemoteCommand struct used to
// set the RDP password on a remote windows machine.
func constructPwdUpdateCommand(ctx context.Context, env evergreen.Environment, h *host.Host, password string) (*jasper.Command, error) {
sshOpts, err := h.GetSSHOptions(env.Settings())
if err != nil {
return nil, errors.WithStack(err)
}
return env.JasperManager().CreateCommand(ctx).
Host(h.Host).User(h.User).
ExtendRemoteArgs(sshOpts...).
Append(fmt.Sprintf("echo -e \"%s\" | passwd", password)), nil
}
func TerminateSpawnHost(ctx context.Context, env evergreen.Environment, host *host.Host, user, reason string) error {
if host.Status == evergreen.HostTerminated {
return errors.New("host is already terminated")
}
if host.Status == evergreen.HostUninitialized {
return host.SetTerminated(ctx, user, "host is an intent host")
}
cloudHost, err := GetCloudHost(ctx, host, env)
if err != nil {
return err
}
if err = cloudHost.TerminateInstance(ctx, user, reason); err != nil {
return err
}
return nil
}
func ModifySpawnHost(ctx context.Context, env evergreen.Environment, host *host.Host, opts host.HostModifyOptions) error {
cloudHost, err := GetCloudHost(ctx, host, env)
if err != nil {
return err
}
if err = cloudHost.ModifyHost(ctx, opts); err != nil {
return err
}
return nil
}
func MakeExtendedSpawnHostExpiration(host *host.Host, extendBy time.Duration) (time.Time, error) {
if err := host.ValidateExpirationExtension(extendBy); err != nil {
return time.Time{}, err
}
newExp := host.ExpirationTime.Add(extendBy)
return newExp, nil
}
func modifySpawnHostProviderSettings(d distro.Distro, settings *evergreen.Settings, region, volumeID string) ([]*birch.Document, error) {
ec2Settings := EC2ProviderSettings{}
if err := ec2Settings.FromDistroSettings(d, region); err != nil {
return nil, errors.Wrapf(err, "getting ec2 provider from distro")
}
if volumeID != "" {
volume, err := host.FindVolumeByID(volumeID)
if err != nil {
return nil, errors.Wrapf(err, "getting volume '%s'", volumeID)
}
ec2Settings.SubnetId, err = getSubnetForZone(settings.Providers.AWS.Subnets, volume.AvailabilityZone)
if err != nil {
return nil, errors.Wrapf(err, "getting subnet for AZ '%s'", volume.AvailabilityZone)
}
}
doc, err := ec2Settings.ToDocument()
if err != nil {
return nil, errors.Wrap(err, "converting EC2 provider settings back to BSON doc")
}
return []*birch.Document{doc}, nil
}