-
Notifications
You must be signed in to change notification settings - Fork 12
/
kill.go
145 lines (123 loc) · 5.28 KB
/
kill.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
// Copyright © 2018 Genome Research Limited
// Author: Sendu Bala <sb10@sanger.ac.uk>.
//
// This file is part of wr.
//
// wr is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// wr is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with wr. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"time"
"github.com/VertebrateResequencing/wr/jobqueue"
"github.com/spf13/cobra"
)
// options for this cmd
var confirmDead bool
var cmdAge string
// killCmd represents the kill command
var killCmd = &cobra.Command{
Use: "kill",
Short: "Kill running commands",
Long: `You can kill commands you've previously added with "wr add" that
are currently running using this command.
After killing commands, there will be a delay before the commands "realise" they
have been killed and actually stop running. At that point they will become
buried and you can "wr remove" them if desired.
Specify one of the flags -f, -l, -i or -a to choose which commands you want to
remove. Amongst those, only running jobs will be affected. If the --confirmdead
option is specified, only "lost contact" jobs will be affected.
-i is the report group (-i) you supplied to "wr add" when you added the job(s)
you want to now kill. Combining with -z lets you kill jobs in multiple report
groups, assuming you have arranged that related groups share some substring.
Alternatively -y lets you specify -i as the internal job id reported during
"wr status".
The file to provide -f is in the format taken by "wr add".
In -f and -l mode you must provide the cwd the commands were set to run in, if
CwdMatters (and must NOT be provided otherwise). Likewise provide the mounts
options that was used when the command was added, if any. You can do this by
using the -c and --mounts/--mounts_json options in -l mode, or by providing the
same file you gave to "wr add" in -f mode.`,
Run: func(cmd *cobra.Command, args []string) {
set := countGetJobArgs()
if set > 1 {
die("-f, -i, -l and -a are mutually exclusive; only specify one of them")
}
if set == 0 {
die("1 of -f, -i, -l or -a is required")
}
timeout := time.Duration(timeoutint) * time.Second
jq := connect(timeout)
var err error
defer func() {
err = jq.Disconnect()
if err != nil {
warn("Disconnecting from the server failed: %s", err)
}
}()
jstate := jobqueue.JobStateRunning
if confirmDead {
jstate = jobqueue.JobStateLost
}
jobs := getJobs(jq, jstate, cmdAll, 0, false, false)
if len(jobs) == 0 {
die("No matching jobs found")
}
var age time.Duration
if cmdAge != "" {
age, err = time.ParseDuration(cmdAge)
if err != nil {
die("--age was not specified correctly: %s", err)
}
}
if age != 0 {
var oldJobs []*jobqueue.Job
for _, job := range jobs {
var thisAge time.Duration
if confirmDead {
thisAge = time.Since(job.EndTime)
} else {
thisAge = job.WallTime()
}
if thisAge >= age {
oldJobs = append(oldJobs, job)
}
}
jobs = oldJobs
}
if len(jobs) == 0 {
die("No matching jobs were older than %s", cmdAge)
}
jes := jobsToJobEssenses(jobs)
killed, err := jq.Kill(jes)
if err != nil {
die("failed to kill desired jobs: %s", err)
}
info("Initiated the termination of %d running commands (out of %d eligible)", killed, len(jobs))
},
}
func init() {
RootCmd.AddCommand(killCmd)
// flags specific to this sub-command
killCmd.Flags().BoolVarP(&cmdAll, "all", "a", false, "kill all running jobs")
killCmd.Flags().StringVarP(&cmdFileStatus, "file", "f", "", "file containing commands you want to kill; - means read from STDIN")
killCmd.Flags().StringVarP(&cmdIDStatus, "identifier", "i", "", "identifier of the commands you want to kill")
killCmd.Flags().BoolVarP(&cmdIDIsSubStr, "search", "z", false, "treat -i as a substring to match against all report groups")
killCmd.Flags().BoolVarP(&cmdIDIsInternal, "internal", "y", false, "treat -i as an internal job id")
killCmd.Flags().StringVarP(&cmdLine, "cmdline", "l", "", "a command line you want to kill")
killCmd.Flags().StringVarP(&cmdCwd, "cwd", "c", "", "working dir that the command(s) specified by -l or -f were set to run in")
killCmd.Flags().StringVarP(&mountJSON, "mount_json", "j", "", "mounts that the command(s) specified by -l or -f were set to use (JSON format)")
killCmd.Flags().StringVar(&mountSimple, "mounts", "", "mounts that the command(s) specified by -l or -f were set to use (simple format)")
killCmd.Flags().BoolVar(&confirmDead, "confirmdead", false, "only confirm that lost contact jobs are dead")
killCmd.Flags().StringVar(&cmdAge, "age", "", "only kill jobs that have been running longer than this (or in --confirmdead mode, that have been lost longer than this). [specify units such as m for minutes or h for hours]")
killCmd.Flags().IntVar(&timeoutint, "timeout", 120, "how long (seconds) to wait to get a reply from 'wr manager'")
}