Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Loading…

Clean up multiprocessing processes on SIGTERM #800

Merged
merged 1 commit into from

2 participants

@grierj

First pass at cleaning up the processes spawned by the salt master
when run in the foreground via some process minder like runit or
daemontools.

This current method is a little inelegant as the cleanup code gets
pulled into the child processes themselves and you waste a few cycles
trying to clean proc objects from the children in the clean_proc() function.

That being said, it works and doesn't orphan processes. I'm open to
better ways to do it, but I didn't want to do any major surgery right
at the moment. I'm not opposed to it, I just had a pressing need, but
I can work on a more integrated solution in the future.

@grierj grierj Clean up multiprocessing processes on SIGTERM
First pass at cleaning up the processes spawned by the salt master
when run in the foreground via some process minder like runit or
daemontools.

This current method is a little inelegant as the cleanup code gets
pulled into the child processes themselves and you waste a few cycles
trying to test the proc objects in the clean_proc() function.

That being said, it works and doesn't orphan processes.
fdfce1b
@thatch45 thatch45 merged commit 9fb95d0 into saltstack:develop
@thatch45
Owner

This is a serious enhancement, thanks Grier!

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Mar 1, 2012
  1. @grierj

    Clean up multiprocessing processes on SIGTERM

    grierj authored
    First pass at cleaning up the processes spawned by the salt master
    when run in the foreground via some process minder like runit or
    daemontools.
    
    This current method is a little inelegant as the cleanup code gets
    pulled into the child processes themselves and you waste a few cycles
    trying to test the proc objects in the clean_proc() function.
    
    That being said, it works and doesn't orphan processes.
This page is out of date. Refresh to see the latest.
Showing with 50 additions and 4 deletions.
  1. +50 −4 salt/master.py
View
54 salt/master.py
@@ -12,6 +12,7 @@
import hashlib
import tempfile
import datetime
+import signal
import multiprocessing
import subprocess
@@ -49,6 +50,26 @@ def prep_jid(opts, load):
return prep_jid(opts['cachedir'], load)
return jid
+def clean_proc(proc, wait_for_kill=1):
+ '''
+ Generic method for cleaning up multiprocessing procs
+ '''
+ # NoneType and other fun stuff need not apply
+ if not proc:
+ return
+ try:
+ waited = 0
+ while proc.is_alive():
+ proc.terminate()
+ waited += 1
+ time.sleep(1)
+ if proc.is_alive() and (waited >= wait_for_kill):
+ log.error(('Process did not die with terminate(): {0}'
+ .format(proc.pid)))
+ os.kill(signal.SIGKILL, proc.pid)
+ except Exception as e:
+ log.debug(e)
+
class SMaster(object):
'''
@@ -121,7 +142,9 @@ def start(self):
Turn on the master server components
'''
log.warn('Starting the Salt Master')
- multiprocessing.Process(target=self._clear_old_jobs).start()
+ clear_old_jobs_proc = multiprocessing.Process(
+ target=self._clear_old_jobs)
+ clear_old_jobs_proc.start()
aes_funcs = AESFuncs(self.opts, self.crypticle)
clear_funcs = ClearFuncs(
self.opts,
@@ -137,12 +160,31 @@ def start(self):
clear_funcs)
reqserv.start_publisher()
+ def sigterm_clean(signum, frame):
+ '''
+ Cleaner method for stoping multiprocessing processes when a SIGTERM
+ is encountered. This is required when running a salt master under
+ a process minder like daemontools
+ '''
+ mypid = os.getpid()
+ log.warn(('Caught signal {0}, stopping the Salt Master'
+ .format(signum)))
+ clean_proc(clear_old_jobs_proc)
+ clean_proc(reqserv.publisher)
+ for proc in reqserv.work_procs:
+ clean_proc(proc)
+ raise SystemExit
+
+ signal.signal(signal.SIGTERM, sigterm_clean)
+
try:
reqserv.run()
except KeyboardInterrupt:
# Shut the master down gracefully on SIGINT
log.warn('Stopping the Salt Master')
raise SystemExit('\nExiting on Ctrl-c')
+ finally:
+ raise SystemExit('Salt Master Stopped')
class Publisher(multiprocessing.Process):
@@ -207,15 +249,19 @@ def __bind(self):
'''
log.info('Setting up the master communication server')
self.clients.bind(self.uri)
+ self.work_procs = []
for ind in range(int(self.opts['worker_threads'])):
- log.info('Starting Salt worker process {0}'.format(ind))
- MWorker(self.opts,
+ self.work_procs.append(MWorker(self.opts,
self.master_key,
self.key,
self.crypticle,
self.aes_funcs,
- self.clear_funcs).start()
+ self.clear_funcs))
+
+ for ind, proc in enumerate(self.work_procs):
+ log.info('Starting Salt worker process {0}'.format(ind))
+ proc.start()
self.workers.bind(self.w_uri)
Something went wrong with that request. Please try again.