Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

master: added live profiling support via yappi

This allows low-overhead adhoc profiling of production servers without
interrupting service.
  • Loading branch information...
commit 1d5c21fd89e99256ba51718ab9b0d7fa5c770be6 1 parent 9888ae3
Alexey Ivanov SaveTheRbtz authored
21 doc/topics/troubleshooting/master.rst
View
@@ -154,6 +154,27 @@ Then pass the signal to the master when it seems to be unresponsive:
When filing an issue or sending questions to the mailing list for a problem
with an unresponsive daemon, be sure to include this information if possible.
+
+Live Salt-Master Profiling
+==========================
+
+When faced with perfomance problems one can turn on master process profiling by
+sending it SIGUSR2.
+
+.. code-block:: bash
+
+ # killall -SIGUSR2 salt-master
+
+This will activate ``yappi`` profiler inside salt-master code, then after some
+time one must send SIGUSR2 again to stop profiling and save results to file. If
+run in foreground salt-master will report filename for the results, which are
+usually located under ``/tmp`` on Unix-based OSes and ``c:\temp`` on windows.
+
+Results can then be analyzed with `kcachegrind`_ or similar tool.
+
+.. _`kcachegrind`: http://kcachegrind.sourceforge.net/html/Home.html
+
+
Commands Time Out or Do Not Return Output
=========================================
1  opt_requirements.txt
View
@@ -1,2 +1,3 @@
mysql-python
timelib
+yappi >= 0.8.2
3  salt/master.py
View
@@ -50,7 +50,7 @@
import salt.utils.verify
import salt.utils.minions
import salt.utils.gzip_util
-from salt.utils.debug import enable_sigusr1_handler, inspect_stack
+from salt.utils.debug import enable_sigusr1_handler, enable_sigusr2_handler, inspect_stack
from salt.exceptions import SaltMasterError, MasterExit
from salt.utils.event import tagify
from salt.pillar import git_pillar
@@ -359,6 +359,7 @@ def start(self):
)
enable_sigusr1_handler()
+ enable_sigusr2_handler()
self.__set_max_open_files()
clear_old_jobs_proc = multiprocessing.Process(
29 salt/utils/debug.py
View
@@ -42,6 +42,28 @@ def _handle_sigusr1(sig, stack):
_makepretty(output, stack)
+def _handle_sigusr2(sig, stack):
+ '''
+ Signal handler for SIGUSR2, only available on Unix-like systems
+ '''
+ try:
+ import yappi
+ except ImportError:
+ return
+ if yappi.is_running():
+ yappi.stop()
+ filename = 'callgrind.salt-{0}-{1}'.format(int(time.time()), os.getpid())
+ destfile = os.path.join(tempfile.gettempdir(), filename)
+ yappi.get_func_stats().save(destfile, type='CALLGRIND')
+ if sys.stderr.isatty():
+ sys.stderr.write('Saved profiling data to: {0}\n'.format(destfile))
+ yappi.clear_stats()
+ else:
+ if sys.stderr.isatty():
+ sys.stderr.write('Profiling started\n')
+ yappi.start()
+
+
def enable_sig_handler(signal_name, handler):
'''
Add signal handler for signal name if it exists on given platform
@@ -61,6 +83,13 @@ def enable_sigusr1_handler():
enable_sig_handler('SIGINFO', _handle_sigusr1)
+def enable_sigusr2_handler():
+ '''
+ Toggle YAPPI profiler
+ '''
+ enable_sig_handler('SIGUSR2', _handle_sigusr2)
+
+
def inspect_stack():
'''
Return a string of which function we are currently in.
Please sign in to comment.
Something went wrong with that request. Please try again.