-
Notifications
You must be signed in to change notification settings - Fork 899
/
runner.rb
200 lines (173 loc) · 5.51 KB
/
runner.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
class MiqCockpitWsWorker::Runner < MiqWorker::Runner
BINDING_ADDRESS = ENV['BINDING_ADDRESS'] || (Rails.env.production? ? "127.0.0.1" : "0.0.0.0")
def find_pids(cmd)
pids = Set.new
Sys::ProcTable.ps.each do |process_struct|
pids << process_struct.pid if cmd =~ (process_struct.try(:cmdline) || "")
end
pids
end
def do_before_work_loop
stop_active_cockpit_ws_processes
start_drb_service
start_cockpit_ws
end
def do_work
check_cockpit_ws
end
def before_exit(_message, _exit_code)
stop_cockpit_ws
stop_drb_service
end
def check_cockpit_ws
if cockpit_ws_alive?
begin
errbuf = @stderr.readpartial(1.megabyte) if @stderr && @stderr.ready?
outbuf = @stdout.readpartial(1.megabyte) if @stdout && @stdout.ready?
rescue EOFError
_log.info("#{log_prefix} got EOF process exiting")
end
outbuf.split("\n").each { |msg| $log.info("cockpit-ws: #{msg.rstrip}") } if outbuf
errbuf.split("\n").each { |msg| $log.error("cockpit-ws: #{msg.rstrip}") } if errbuf
else
_log.info("#{log_prefix} Cockpit-ws Process gone. Restarting...")
start_cockpit_ws
end
end
def start_cockpit_ws
raise _("Cannot call start_cockpit_ws if a process already exists") if cockpit_ws_alive?
_log.info("#{log_prefix} Starting cockpit-ws Process")
start_cockpit_ws_process
_log.info("#{log_prefix} Started cockpit-ws Process")
end
def start_cockpit_ws_process
@pid, @stdout, @stderr = cockpit_ws_run
rescue => err
@stdout.close if @stdout
@stderr.close if @stderr
_log.error("#{log_prefix} cockpit-ws Process aborted because [#{err.message}]")
_log.log_backtrace(err)
end
def stop_cockpit_ws
if cockpit_ws_alive?
_log.info("#{log_prefix} Shutting down cockpit-ws process...pid=#{@pid}")
stop_cockpit_ws_process
_log.info("#{log_prefix} Shutting down cockpit-ws process...Complete")
end
end
def stop_cockpit_ws_process
return unless @pid
@stdout&.close
@stderr&.close
Process.kill("TERM", @pid)
wait_on_cockpit_ws
end
# Waits for a cockpit-ws process to stop. The process is expected to be
# in the act of shutting down, and thus it will wait 5 minutes
# before issuing a kill.
def wait_on_cockpit_ws(pid = nil)
pid ||= @pid
# TODO: Use Process.waitpid or one of its async variants
begin
Timeout.timeout(5.minutes.to_i) do
loop do
break unless process_alive?(pid)
sleep 1
heartbeat
end
end
rescue Timeout::Error
_log.info("#{log_prefix} Killing cockpit-ws process with pid=#{pid}")
Process.kill(9, pid)
end
begin
_, status = Process.waitpid2(pid)
_log.info("#{log_prefix} cockpit-ws Process with pid=#{pid} exited with a status=#{status}")
rescue Errno::ECHILD
_log.info("#{log_prefix} cockpit-ws Process with pid=#{pid} exited")
end
reset_process_info
end
def reset_process_info
@pid = @stdout = @stderr = nil
end
def find_cockpit_ws_processes
find_pids(/cockpit-ws --port/)
end
def stop_active_cockpit_ws_processes
find_cockpit_ws_processes.each do |pid|
_log.info("#{log_prefix} Killing active cockpit-ws process with pid=#{pid}")
Process.kill("TERM", pid)
wait_on_cockpit_ws(pid)
end
end
def cockpit_ws_alive?
if process_alive?(@pid)
true
else
reset_process_info
false
end
end
def process_alive?(pid)
return false if pid.nil?
process_struct = Sys::ProcTable.ps(pid)
return false if process_struct.nil?
return true if process_struct.state != "Z"
_log.info("#{log_prefix} waiting to die")
zombie_pid, status = Process.waitpid2(pid)
_log.info("#{log_prefix} cockpit-ws Process with pid=#{zombie_pid} exited with a status=#{status}")
false
end
def cockpit_ws_run
_log.info("#{log_prefix} cockpit-ws process starting")
opts = @worker_settings
cockpit_ws = MiqCockpit::WS.new(opts)
cockpit_ws.save_config
require "open3"
env = {
"XDG_CONFIG_DIRS" => cockpit_ws.config_dir,
"DRB_URI" => @drb_uri
}
_log.info("Starting cockpit-ws process with command: #{cockpit_ws.command(BINDING_ADDRESS)} ")
_log.info("Cockpit environment #{env} ")
stdin, stdout, stderr, wait_thr = Bundler.with_clean_env do
Open3.popen3(env, cockpit_ws.command(BINDING_ADDRESS), :unsetenv_others => true)
end
stdin&.close
if wait_thr
_log.info("#{log_prefix} cockpit-ws process started - pid=#{wait_thr.pid}")
return wait_thr.pid, stdout, stderr
else
raise "Cockpit-ws process failed to start"
end
end
def check_drb_service
alive = @drb_server ? @drb_server.alive? : false
unless alive
start_drb_service
end
end
def start_drb_service
require 'drb'
require 'drb/acl'
stop_drb_service
acl = ACL.new(%w(deny all allow 127.0.0.1/32))
if @drb_uri
@drb_server = DRb::DRbServer.new(@drb_uri, MiqCockpitWsWorker::Authenticator, acl)
else
require 'tmpdir'
Dir::Tmpname.create("cockpit", nil) do |path|
@drb_server = DRb::DRbServer.new("drbunix://#{path}", MiqCockpitWsWorker::Authenticator, acl)
FileUtils.chmod(0o750, path)
end
end
@drb_uri = @drb_server.uri
_log.info("#{log_prefix} Started drb Process at #{@drb_uri}")
end
def stop_drb_service
@drb_server.stop_service if @drb_server
@drb_server = nil
_log.info("#{log_prefix} stopped drb Process at #{@drb_uri}")
end
end