From 1b8e0d048301d9272f699f9534f7f7e8c39402b2 Mon Sep 17 00:00:00 2001 From: Mathieu Le Marec - Pasquet Date: Sun, 24 May 2015 15:52:55 +0200 Subject: [PATCH 1/8] Refactor test.rand_str to reuse later Signed-off-by: Mathieu Le Marec - Pasquet --- salt/modules/test.py | 16 ++++++++++++---- salt/utils/__init__.py | 10 ++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/salt/modules/test.py b/salt/modules/test.py index fc400f1b4e29..5bcb9863f6c7 100644 --- a/salt/modules/test.py +++ b/salt/modules/test.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- ''' Module for running arbitrary tests ''' @@ -14,6 +13,7 @@ # Import Salt libs import salt +import salt.utils import salt.version import salt.loader import salt.ext.six as six @@ -469,18 +469,26 @@ def opts_pkg(): return ret -def rand_str(size=9999999999): +def rand_str(size=9999999999, hash_type='md5'): ''' Return a random string + size + size of the string to generate + hash_type + hash type to use + + .. versionadded:: 2015.5.2 + CLI Example: .. code-block:: bash salt '*' test.rand_str ''' - hasher = getattr(hashlib, __opts__.get('hash_type', 'md5')) - return hasher(str(random.SystemRandom().randint(0, size))).hexdigest() + return salt.utils.rand_str( + hash_type= __opts__.get('hash_type', hash_type), + size=size) def exception(message='Test Exception'): diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index 11402444bcb3..d398dcaeef74 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -437,6 +437,16 @@ def profiled_func(*args, **kwargs): return proffunc +def rand_str(size=9999999999, hash_type=None): + ''' + Return a random string + ''' + if not hash_type: + hash_type = 'md5' + hasher = getattr(hashlib, hash_type) + return hasher(str(random.SystemRandom().randint(0, size))).hexdigest() + + def which(exe=None): ''' Python clone of /usr/bin/which From 6b01973b86d9031172dd7be2d4c0ec1cc185c2fb Mon Sep 17 00:00:00 2001 From: Mathieu Le Marec - Pasquet Date: Sun, 24 May 2015 16:43:40 +0200 Subject: [PATCH 2/8] test.rand_str: better kwargs handling --- salt/modules/test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/salt/modules/test.py b/salt/modules/test.py index 5bcb9863f6c7..9dca7e5322c7 100644 --- a/salt/modules/test.py +++ b/salt/modules/test.py @@ -469,7 +469,7 @@ def opts_pkg(): return ret -def rand_str(size=9999999999, hash_type='md5'): +def rand_str(size=9999999999, hash_type=None): ''' Return a random string @@ -486,9 +486,9 @@ def rand_str(size=9999999999, hash_type='md5'): salt '*' test.rand_str ''' - return salt.utils.rand_str( - hash_type= __opts__.get('hash_type', hash_type), - size=size) + if not hash_type: + hash_type = __opts__.get('hash_type', 'md5') + return salt.utils.rand_str(hash_type=hash_type, size=size) def exception(message='Test Exception'): From 96b759300a32e6115f1b6817c5a98272280181f2 Mon Sep 17 00:00:00 2001 From: Mathieu Le Marec - Pasquet Date: Sun, 24 May 2015 15:53:31 +0200 Subject: [PATCH 3/8] Salt call profiling support Signed-off-by: Mathieu Le Marec - Pasquet --- salt/cli/caller.py | 14 +++++++++- salt/utils/__init__.py | 58 +++++++++++++++++++++++++++++++++++++++++- salt/utils/parsers.py | 32 ++++++++++++++++++++++- 3 files changed, 101 insertions(+), 3 deletions(-) diff --git a/salt/cli/caller.py b/salt/cli/caller.py index 801f18207bba..51a143dd9cb7 100644 --- a/salt/cli/caller.py +++ b/salt/cli/caller.py @@ -6,6 +6,8 @@ # Import python libs from __future__ import absolute_import, print_function +import cProfile +import datetime import os import sys import time @@ -27,6 +29,8 @@ from salt.utils import is_windows from salt.utils import print_cli from salt.utils import kinds +from salt.utils import activate_profile +from salt.utils import output_profile from salt.cli import daemons try: @@ -123,8 +127,16 @@ def run(self): ''' Execute the salt call logic ''' + profiling_enabled = self.opts.get('profiling_enabled', False) try: - ret = self.call() + pr = activate_profile(profiling_enabled) + try: + ret = self.call() + finally: + output_profile(pr, + stats_path=self.opts.get('profiling_path', + '/tmp/stats'), + stop=True) out = ret.get('out', 'nested') if self.opts['metadata']: print_ret = ret diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index d398dcaeef74..35807abbcbd6 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -23,13 +23,16 @@ import shlex import shutil import socket +import cProfile import stat import sys +import pstats import tempfile import time import types import warnings import string +import subprocess # Import 3rd-party libs import salt.ext.six as six @@ -420,7 +423,6 @@ def profile_func(filename=None): ''' def proffunc(fun): def profiled_func(*args, **kwargs): - import cProfile logging.info('Profiling function {0}'.format(fun.__name__)) try: profiler = cProfile.Profile() @@ -535,6 +537,60 @@ def which_bin(exes): return None +def activate_profile(test=True): + pr = None + if test: + pr = cProfile.Profile() + pr.enable() + return pr + + +def output_profile(pr, stats_path='/tmp/stats', stop=False, id_=None): + if pr is not None: + try: + pr.disable() + if not os.path.isdir(stats_path): + os.makedirs(stats_path) + date = datetime.datetime.now().isoformat() + if id_ is None: + id_ = rand_str(size=32) + ficp = os.path.join(stats_path, '{0}.{1}.pstats'.format(id_, date)) + fico = os.path.join(stats_path, '{0}.{1}.dot'.format(id_, date)) + ficn = os.path.join(stats_path, '{0}.{1}.stats'.format(id_, date)) + if not os.path.exists(ficp): + pr.dump_stats(ficp) + with open(ficn, 'w') as fic: + pstats.Stats(pr, stream=fic).sort_stats('cumulative') + log.info('PROFILING: {0} generated'.format(ficp)) + log.info('PROFILING (cumulative): {0} generated'.format(ficn)) + pyprof = which('pyprof2calltree') + cmd = [pyprof, '-i', ficp, '-o', fico] + if pyprof: + failed = False + try: + pro = subprocess.Popen( + cmd, shell=False, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except OSError: + failed = True + if pro.returncode: + failed = True + if failed: + log.error('PROFILING (dot problem') + else: + log.info('PROFILING (dot): {0} generated'.format(fico)) + log.trace('pyprof2calltree output:') + log.trace(pro.stdout.read().strip() + + pro.stderr.read().strip()) + else: + log.info('You can run {0} for additional stats.'.format(cmd)) + finally: + if not stop: + pr.enable() + return pr + + + def list_files(directory): ''' Return a list of all files found under directory diff --git a/salt/utils/parsers.py b/salt/utils/parsers.py index 19666dfdd50e..0bd31053f2e4 100644 --- a/salt/utils/parsers.py +++ b/salt/utils/parsers.py @@ -1377,6 +1377,35 @@ def _mixin_after_parsed(self): ) +class ProfilingPMixIn(six.with_metaclass(MixInMeta, object)): + _mixin_prio_ = 130 + + def _mixin_setup(self): + group = self.profiling_group = optparse.OptionGroup( + self, + 'Profiling support', + # Include description here as a string + ) + + group.add_option( + '--profiling-path', + dest='profiling_path', + default='/tmp/stats', + help=('Folder that will hold all' + ' Stats generations path (/tmp/stats)') + ) + group.add_option( + '--enable-profiling', + dest='profiling_enabled', + default=False, + action='store_true', + metavar=' ', + help=('Enable generating profiling stats' + ' in /tmp/stats (--profiling-path)') + ) + self.add_option_group(group) + + class CloudCredentialsMixIn(six.with_metaclass(MixInMeta, object)): _mixin_prio_ = 30 @@ -2053,7 +2082,8 @@ class SaltCallOptionParser(six.with_metaclass(OptionParserMeta, OutputOptionsMixIn, HardCrashMixin, SaltfileMixIn, - ArgsStdinMixIn)): + ArgsStdinMixIn, + ProfilingPMixIn)): description = ('Salt call is used to execute module functions locally ' 'on a minion') From 12734be0e982c490a78e16571b57f07dff866f7e Mon Sep 17 00:00:00 2001 From: Mathieu Le Marec - Pasquet Date: Sun, 24 May 2015 16:33:53 +0200 Subject: [PATCH 4/8] Support for profiling in salt-run Signed-off-by: Mathieu Le Marec - Pasquet --- salt/cli/run.py | 12 +++++++++++- salt/utils/parsers.py | 3 ++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/salt/cli/run.py b/salt/cli/run.py index a6cef7d8c42f..4718cf2267c3 100644 --- a/salt/cli/run.py +++ b/salt/cli/run.py @@ -5,6 +5,8 @@ import os from salt.utils import parsers +from salt.utils import activate_profile +from salt.utils import output_profile from salt.utils.verify import check_user from salt.exceptions import SaltClientError @@ -23,6 +25,7 @@ def run(self): # Setup file logging! self.setup_logfile_logger() + profiling_enabled = self.options.profiling_enabled runner = salt.runner.Runner(self.config) if self.options.doc: @@ -33,6 +36,13 @@ def run(self): # someone tries to use the runners via the python API try: if check_user(self.config['user']): - runner.run() + pr = activate_profile(profiling_enabled) + try: + runner.run() + finally: + output_profile( + pr, + stats_path=self.options.profiling_path, + stop=True) except SaltClientError as exc: raise SystemExit(str(exc)) diff --git a/salt/utils/parsers.py b/salt/utils/parsers.py index 0bd31053f2e4..e7a7a2661451 100644 --- a/salt/utils/parsers.py +++ b/salt/utils/parsers.py @@ -2286,7 +2286,8 @@ class SaltRunOptionParser(six.with_metaclass(OptionParserMeta, HardCrashMixin, SaltfileMixIn, OutputOptionsMixIn, - ArgsStdinMixIn)): + ArgsStdinMixIn, + ProfilingPMixIn)): default_timeout = 1 From d07949d23b89cde146dbb2984cee872b7e0bc0b9 Mon Sep 17 00:00:00 2001 From: Mathieu Le Marec - Pasquet Date: Sun, 24 May 2015 16:38:35 +0200 Subject: [PATCH 5/8] Gate cProfile Signed-off-by: Mathieu Le Marec - Pasquet --- salt/cli/caller.py | 3 +-- salt/utils/__init__.py | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/salt/cli/caller.py b/salt/cli/caller.py index 51a143dd9cb7..501154bf9dbc 100644 --- a/salt/cli/caller.py +++ b/salt/cli/caller.py @@ -6,8 +6,7 @@ # Import python libs from __future__ import absolute_import, print_function -import cProfile -import datetime + import os import sys import time diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index 35807abbcbd6..96db6d6efb06 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -23,7 +23,6 @@ import shlex import shutil import socket -import cProfile import stat import sys import pstats @@ -45,6 +44,13 @@ from stat import S_IMODE # pylint: enable=import-error,redefined-builtin + +try: + import cProfile + HAS_CPROFILE = True +except ImportError: + HAS_CPROFILE = False + # Try to load pwd, fallback to getpass if unsuccessful # Import 3rd-party libs try: @@ -540,13 +546,16 @@ def which_bin(exes): def activate_profile(test=True): pr = None if test: - pr = cProfile.Profile() - pr.enable() + if HAS_CPROFILE: + pr = cProfile.Profile() + pr.enable() + else: + log.error('cProfile is not available on your platform') return pr def output_profile(pr, stats_path='/tmp/stats', stop=False, id_=None): - if pr is not None: + if pr is not None and HAS_CPROFILE: try: pr.disable() if not os.path.isdir(stats_path): From 3dae4ca1f8998b1045f98f6a3cc4de6ffc7a051c Mon Sep 17 00:00:00 2001 From: Mathieu Le Marec - Pasquet Date: Sun, 24 May 2015 16:47:05 +0200 Subject: [PATCH 6/8] parsers: fix --enable-profiling option Signed-off-by: Mathieu Le Marec - Pasquet --- salt/utils/parsers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/salt/utils/parsers.py b/salt/utils/parsers.py index e7a7a2661451..f1c3005aa5c8 100644 --- a/salt/utils/parsers.py +++ b/salt/utils/parsers.py @@ -1399,7 +1399,6 @@ def _mixin_setup(self): dest='profiling_enabled', default=False, action='store_true', - metavar=' ', help=('Enable generating profiling stats' ' in /tmp/stats (--profiling-path)') ) From 2931df00fc3f5398986a25a0fa5e198cbe471a3a Mon Sep 17 00:00:00 2001 From: Mathieu Le Marec - Pasquet Date: Mon, 25 May 2015 01:20:29 +0200 Subject: [PATCH 7/8] lint Signed-off-by: Mathieu Le Marec - Pasquet --- salt/modules/test.py | 1 - salt/utils/__init__.py | 1 - 2 files changed, 2 deletions(-) diff --git a/salt/modules/test.py b/salt/modules/test.py index 9dca7e5322c7..bcd5f98c9235 100644 --- a/salt/modules/test.py +++ b/salt/modules/test.py @@ -8,7 +8,6 @@ import sys import time import traceback -import hashlib import random # Import Salt libs diff --git a/salt/utils/__init__.py b/salt/utils/__init__.py index 96db6d6efb06..5e6db1363706 100644 --- a/salt/utils/__init__.py +++ b/salt/utils/__init__.py @@ -599,7 +599,6 @@ def output_profile(pr, stats_path='/tmp/stats', stop=False, id_=None): return pr - def list_files(directory): ''' Return a list of all files found under directory From b093d32f998d7f6fc9e40164d053c9c0caa98649 Mon Sep 17 00:00:00 2001 From: Mathieu Le Marec - Pasquet Date: Fri, 29 May 2015 10:23:45 +0200 Subject: [PATCH 8/8] lint Signed-off-by: Mathieu Le Marec - Pasquet --- salt/modules/test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/modules/test.py b/salt/modules/test.py index bcd5f98c9235..a5f54ac88c98 100644 --- a/salt/modules/test.py +++ b/salt/modules/test.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- ''' Module for running arbitrary tests '''