master: added live profiling support via yappi

This allows low-overhead adhoc profiling of production servers without
interrupting service.

doc/topics/troubleshooting/master.rst

@@ -154,6 +154,27 @@ Then pass the signal to the master when it seems to be unresponsive:
 When filing an issue or sending questions to the mailing list for a problem
 with an unresponsive daemon, be sure to include this information if possible.
 
+
+Live Salt-Master Profiling
+==========================
+
+When faced with perfomance problems one can turn on master process profiling by
+sending it SIGUSR2.
+
+.. code-block:: bash
+
+    # killall -SIGUSR2 salt-master
+
+This will activate ``yappi`` profiler inside salt-master code, then after some
+time one must send SIGUSR2 again to stop profiling and save results to file. If
+run in foreground salt-master will report filename for the results, which are
+usually located under ``/tmp`` on Unix-based OSes and ``c:\temp`` on windows.
+
+Results can then be analyzed with `kcachegrind`_ or similar tool.
+
+.. _`kcachegrind`: http://kcachegrind.sourceforge.net/html/Home.html
+
+
 Commands Time Out or Do Not Return Output
 =========================================
 

opt_requirements.txt

@@ -1,2 +1,3 @@
 mysql-python
 timelib
+yappi >= 0.8.2

salt/master.py

@@ -50,7 +50,7 @@
 import salt.utils.verify
 import salt.utils.minions
 import salt.utils.gzip_util
-from salt.utils.debug import enable_sigusr1_handler, inspect_stack
+from salt.utils.debug import enable_sigusr1_handler, enable_sigusr2_handler, inspect_stack
 from salt.exceptions import SaltMasterError, MasterExit
 from salt.utils.event import tagify
 from salt.pillar import git_pillar
@@ -359,6 +359,7 @@ def start(self):
         )
 
         enable_sigusr1_handler()
+        enable_sigusr2_handler()
 
         self.__set_max_open_files()
         clear_old_jobs_proc = multiprocessing.Process(

salt/utils/debug.py

@@ -42,6 +42,28 @@ def _handle_sigusr1(sig, stack):
             _makepretty(output, stack)
 
 
+def _handle_sigusr2(sig, stack):
+    '''
+    Signal handler for SIGUSR2, only available on Unix-like systems
+    '''
+    try:
+        import yappi
+    except ImportError:
+        return
+    if yappi.is_running():
+        yappi.stop()
+        filename = 'callgrind.salt-{0}-{1}'.format(int(time.time()), os.getpid())
+        destfile = os.path.join(tempfile.gettempdir(), filename)
+        yappi.get_func_stats().save(destfile, type='CALLGRIND')
+        if sys.stderr.isatty():
+            sys.stderr.write('Saved profiling data to: {0}\n'.format(destfile))
+        yappi.clear_stats()
+    else:
+        if sys.stderr.isatty():
+            sys.stderr.write('Profiling started\n')
+        yappi.start()
+
+
 def enable_sig_handler(signal_name, handler):
     '''
     Add signal handler for signal name if it exists on given platform
@@ -61,6 +83,13 @@ def enable_sigusr1_handler():
     enable_sig_handler('SIGINFO', _handle_sigusr1)
 
 
+def enable_sigusr2_handler():
+    '''
+    Toggle YAPPI profiler
+    '''
+    enable_sig_handler('SIGUSR2', _handle_sigusr2)
+
+
 def inspect_stack():
     '''
     Return a string of which function we are currently in.