-
Notifications
You must be signed in to change notification settings - Fork 12
/
serverWebI.go
515 lines (470 loc) · 14.7 KB
/
serverWebI.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
// Copyright © 2016-2018 Genome Research Limited
// Author: Sendu Bala <sb10@sanger.ac.uk>.
//
// This file is part of wr.
//
// wr is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// wr is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with wr. If not, see <http://www.gnu.org/licenses/>.
package jobqueue
// This file contains the web interface code of the server.
import (
"net/http"
"strings"
"sync"
"github.com/VertebrateResequencing/wr/internal"
"github.com/VertebrateResequencing/wr/queue"
"github.com/gorilla/websocket"
)
// jstatusReq is what the status webpage sends us to ask for info about jobs.
type jstatusReq struct {
// possible Requests are:
// current = get count info for every job in every RepGroup in the cmds
// queue.
// details = get example job details for jobs in the RepGroup, grouped by
// having the same Status, Exitcode and FailReason.
// retry = retry buried jobs.
// remove = remove non-running jobs.
// kill = kill running jobs or confirm lost jobs are dead.
// confirmBadServer = confirm that the server with ID ServerID is bad.
// dismissMsg = dismiss the given Msg.
Request string
// sending Key means "give me detailed info about this single job", and
// modifies retry, remove and kill to only work on this job
Key string
// sending RepGroup means "send me limited info about the jobs with this
// RepGroup", and modifies retry, remove and kill to work on all jobs with
// the given RepGroup, ExitCode and FailReason
RepGroup string
State JobState // A Job.State to limit RepGroup by in details mode
Exitcode int
FailReason string
ServerID string // required argument for confirmBadServer
Msg string // required argument for dismissMsg
}
// JStatus is the job info we send to the status webpage (only real difference
// to Job is that some of the values are converted to easy-to-display forms).
type JStatus struct {
Key string
RepGroup string
DepGroups []string
Dependencies []string
Cmd string
State JobState
Cwd string
CwdBase string
HomeChanged bool
Behaviours string
Mounts string
MonitorDocker string
// ExpectedRAM is in Megabytes.
ExpectedRAM int
// ExpectedTime is in seconds.
ExpectedTime float64
// RequestedDisk is in Gigabytes.
RequestedDisk int
OtherRequests []string
Cores int
PeakRAM int
Exited bool
Exitcode int
FailReason string
Pid int
Host string
HostID string
HostIP string
Walltime float64
CPUtime float64
Started int64
Ended int64
StdErr string
StdOut string
Env []string
Attempts uint32
Similar int
}
// webInterfaceStatic is a http handler for our static documents in static.go
// (which in turn come from the static folder in the git repository). static.go
// is auto-generated by:
// $ esc -pkg jobqueue -prefix static -private -o jobqueue/static.go static
func webInterfaceStatic(s *Server) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
// our home page is /status.html
path := r.URL.Path
if path == "/" || path == "/status" {
path = "/status.html"
ok := s.httpAuthorized(w, r)
if !ok {
return
}
}
// during development, to avoid having to rebuild and restart manager on
// every change to a file in static dir, do:
// $ esc -pkg jobqueue -prefix $GOPATH/src/github.com/VertebrateResequencing/wr/static -private -o jobqueue/static.go $GOPATH/src/github.com/VertebrateResequencing/wr/static
// and set the boolean to true. Don't forget to rerun esc without the abs
// paths and change the boolean back to false before any commit!
doc, err := _escFSByte(false, path)
if err != nil {
http.NotFound(w, r)
return
}
if strings.HasPrefix(path, "/js") {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
} else if strings.HasPrefix(path, "/css") {
w.Header().Set("Content-Type", "text/css; charset=utf-8")
} else if strings.HasPrefix(path, "/fonts") {
if strings.HasSuffix(path, ".eot") {
w.Header().Set("Content-Type", "application/vnd.ms-fontobject")
} else if strings.HasSuffix(path, ".svg") {
w.Header().Set("Content-Type", "image/svg+xml")
} else if strings.HasSuffix(path, ".ttf") {
w.Header().Set("Content-Type", "application/x-font-truetype")
} else if strings.HasSuffix(path, ".woff") {
w.Header().Set("Content-Type", "application/font-woff")
} else if strings.HasSuffix(path, ".woff2") {
w.Header().Set("Content-Type", "application/font-woff2")
}
} else if strings.HasSuffix(path, "favicon.ico") {
w.Header().Set("Content-Type", "image/x-icon")
}
_, err = w.Write(doc)
if err != nil {
s.Error("web interface static document write failed", "err", err)
}
}
}
// webSocket upgrades a http connection to a websocket
func webSocket(w http.ResponseWriter, r *http.Request) (*websocket.Conn, bool) {
var upgrader = websocket.Upgrader{
ReadBufferSize: 1024,
WriteBufferSize: 1024,
}
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
http.Error(w, "Could not open websocket connection", http.StatusBadRequest)
return conn, false
}
return conn, true
}
// webInterfaceStatusWS reads from and writes to the websocket on the status
// webpage
func webInterfaceStatusWS(s *Server) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
ok := s.httpAuthorized(w, r)
if !ok {
return
}
conn, ok := webSocket(w, r)
if !ok {
s.Error("Failed to set up websocket", "Host", r.Host)
return
}
writeMutex := &sync.Mutex{}
// when the server shuts down it will close our conn, ending the main
// goroutine
storedName := s.storeWebSocketConnection(conn)
// when the main goroutine closes we will end all the others
stopper := make(chan bool)
// go routine to read client requests and respond to them
go func(conn *websocket.Conn, connStorageName string, stop chan bool) {
// log panics and die
defer internal.LogPanic(s.Logger, "jobqueue websocket client handling", true)
defer func() {
s.closeWebSocketConnection(connStorageName)
// stop the other goroutines
close(stop)
}()
for {
req := jstatusReq{}
errr := conn.ReadJSON(&req)
if errr != nil {
// browser was refreshed or server shutdown
break
}
switch {
case req.Request != "":
switch req.Request {
case "current":
// get all current jobs
jobs := s.getJobsCurrent(0, "", false, false)
writeMutex.Lock()
err := webInterfaceStatusSendGroupStateCount(conn, "+all+", jobs)
if err != nil {
writeMutex.Unlock()
break
}
// for each different RepGroup amongst these jobs,
// send the job state counts
repGroups := make(map[string][]*Job)
for _, job := range jobs {
repGroups[job.RepGroup] = append(repGroups[job.RepGroup], job)
}
failed := false
for repGroup, jobs := range repGroups {
complete, _, qerr := s.getCompleteJobsByRepGroup(repGroup)
if qerr != "" {
failed = true
break
}
jobs = append(jobs, complete...)
err := webInterfaceStatusSendGroupStateCount(conn, repGroup, jobs)
if err != nil {
failed = true
break
}
}
// also send details of dead servers
for _, bs := range s.getBadServers() {
s.badServerCaster.Send(bs)
}
// and of scheduler messages
s.simutex.RLock()
for _, si := range s.schedIssues {
s.schedCaster.Send(si)
}
s.simutex.RUnlock()
writeMutex.Unlock()
if failed {
break
}
case "details":
// *** probably want to take the count as a req option,
// so user can request to see more than just 1 job per
// State+Exitcode+FailReason
jobs, _, errstr := s.getJobsByRepGroup(req.RepGroup, false, 1, req.State, true, true)
if errstr == "" && len(jobs) > 0 {
writeMutex.Lock()
failed := false
for _, job := range jobs {
status := job.ToStatus()
status.RepGroup = req.RepGroup // since we want to return the group the user asked for, not the most recent group the job was made for
err := conn.WriteJSON(status)
if err != nil {
failed = true
break
}
}
writeMutex.Unlock()
if failed {
break
}
}
case "retry":
jobs := s.reqToJobs(req, []queue.ItemState{queue.ItemStateBury})
for _, job := range jobs {
err := s.q.Kick(job.Key())
if err != nil {
continue
}
job.UntilBuried = job.Retries + 1
}
case "remove":
jobs := s.reqToJobs(req, []queue.ItemState{queue.ItemStateBury, queue.ItemStateDelay, queue.ItemStateDependent, queue.ItemStateReady})
var toDelete []string
for _, job := range jobs {
key := job.Key()
// we can't allow the removal of jobs that have
// dependencies, as *queue would regard that as
// satisfying the dependency and downstream jobs
// would start
hasDeps, err := s.q.HasDependents(key)
if err != nil || hasDeps {
continue
}
err = s.q.Remove(key)
if err != nil {
s.Warn("failed to remove job", "cmd", job.Cmd, "err", err)
continue
}
s.db.deleteLiveJob(key)
s.Debug("removed job", "cmd", job.Cmd)
toDelete = append(toDelete, key)
if job.State == JobStateReady {
s.decrementGroupCount(job.schedulerGroup)
}
}
s.rpl.Lock()
for _, key := range toDelete {
delete(s.rpl.lookup[req.RepGroup], key)
}
s.rpl.Unlock()
case "kill":
jobs := s.reqToJobs(req, []queue.ItemState{queue.ItemStateRun})
for _, job := range jobs {
_, err := s.killJob(job.Key())
if err != nil {
s.Warn("web interface kill job failed", "err", err)
}
}
case "confirmBadServer":
if req.ServerID != "" {
s.bsmutex.Lock()
server := s.badServers[req.ServerID]
delete(s.badServers, req.ServerID)
s.bsmutex.Unlock()
if server != nil && server.IsBad() {
err := server.Destroy()
if err != nil {
s.Warn("web interface confirm bad server destruction failed", "err", err)
}
}
}
case "dismissMsg":
if req.Msg != "" {
s.simutex.Lock()
delete(s.schedIssues, req.Msg)
s.simutex.Unlock()
}
default:
continue
}
case req.Key != "":
jobs, _, errstr := s.getJobsByKeys([]string{req.Key}, true, true)
if errstr == "" && len(jobs) == 1 {
status := jobs[0].ToStatus()
writeMutex.Lock()
err := conn.WriteJSON(status)
writeMutex.Unlock()
if err != nil {
break
}
}
}
}
}(conn, storedName, stopper)
// go routines to push changes to the client
go func(conn *websocket.Conn, stop chan bool) {
// log panics and die
defer internal.LogPanic(s.Logger, "jobqueue websocket status updating", true)
statusReceiver := s.statusCaster.Join()
defer statusReceiver.Close()
for {
select {
case <-stop:
return
case status := <-statusReceiver.In:
writeMutex.Lock()
err := conn.WriteJSON(status)
writeMutex.Unlock()
if err != nil {
s.Warn("status updater failed to send JSON to client", "err", err)
return
}
}
}
}(conn, stopper)
go func(conn *websocket.Conn, stop chan bool) {
defer internal.LogPanic(s.Logger, "jobqueue websocket bad server updating", true)
badserverReceiver := s.badServerCaster.Join()
defer badserverReceiver.Close()
for {
select {
case <-stop:
return
case server := <-badserverReceiver.In:
writeMutex.Lock()
err := conn.WriteJSON(server)
writeMutex.Unlock()
if err != nil {
s.Warn("bad server caster failed to send JSON to client", "err", err)
return
}
}
}
}(conn, stopper)
go func(conn *websocket.Conn, stop chan bool) {
defer internal.LogPanic(s.Logger, "jobqueue websocket scheduler issue updating", true)
schedIssueReceiver := s.schedCaster.Join()
defer schedIssueReceiver.Close()
for {
select {
case <-stop:
return
case si := <-schedIssueReceiver.In:
writeMutex.Lock()
err := conn.WriteJSON(si)
writeMutex.Unlock()
if err != nil {
s.Warn("scheduler issues caster failed to send JSON to client", "err", err)
return
}
}
}
}(conn, stopper)
}
}
// reqToJobs takes a request from the status webpage and returns the requested
// jobs.
func (s *Server) reqToJobs(req jstatusReq, allowedItemStates []queue.ItemState) []*Job {
allowed := make(map[queue.ItemState]bool)
for _, is := range allowedItemStates {
allowed[is] = true
}
var jobs []*Job
if req.RepGroup != "" {
s.rpl.RLock()
defer s.rpl.RUnlock()
for key := range s.rpl.lookup[req.RepGroup] {
item, err := s.q.Get(key)
if item == nil || err != nil {
continue
}
stats := item.Stats()
if allowed[stats.State] {
job := item.Data.(*Job)
job.Lock()
job.State = s.itemStateToJobState(stats.State, job.Lost)
if job.Exitcode == req.Exitcode && job.FailReason == req.FailReason {
jobs = append(jobs, job)
}
job.Unlock()
}
}
} else if req.Key != "" {
item, err := s.q.Get(req.Key)
if item == nil || err != nil {
return nil
}
stats := item.Stats()
if allowed[stats.State] {
job := item.Data.(*Job)
job.Lock()
job.State = s.itemStateToJobState(stats.State, job.Lost)
job.Unlock()
jobs = append(jobs, job)
}
}
return jobs
}
// webInterfaceStatusSendGroupStateCount sends the per-repgroup state counts
// to the status webpage websocket
func webInterfaceStatusSendGroupStateCount(conn *websocket.Conn, repGroup string, jobs []*Job) error {
stateCounts := make(map[JobState]int)
for _, job := range jobs {
var state JobState
// for display simplicity purposes, merge reserved in to running
switch job.State {
case JobStateReserved, JobStateRunning:
state = JobStateRunning
default:
state = job.State
}
stateCounts[state]++
}
for to, count := range stateCounts {
err := conn.WriteJSON(&jstateCount{repGroup, JobStateNew, to, count})
if err != nil {
return err
}
}
return nil
}