From 40759953e175ba374fda85dc57ade46b20455ba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?xavier=20dupr=C3=A9?= Date: Sat, 9 Oct 2021 00:56:07 +0200 Subject: [PATCH] Add function profile2df --- _unittests/ut_pycode/test_profiling.py | 8 ++- .../jenkinshelper/yaml_helper.py | 3 +- src/pyquickhelper/pycode/profiling.py | 59 +++++++++++++++---- .../sphinxext/sphinx_md_builder.py | 2 +- .../sphinxext/sphinx_rst_builder.py | 4 +- 5 files changed, 57 insertions(+), 19 deletions(-) diff --git a/_unittests/ut_pycode/test_profiling.py b/_unittests/ut_pycode/test_profiling.py index 24908fbc..58b8311d 100644 --- a/_unittests/ut_pycode/test_profiling.py +++ b/_unittests/ut_pycode/test_profiling.py @@ -6,11 +6,10 @@ import unittest import warnings import pandas - from pyquickhelper.pycode import ExtTestCase from pyquickhelper.pandashelper import df2rst from pyquickhelper import __file__ as rootfile -from pyquickhelper.pycode.profiling import profile +from pyquickhelper.pycode.profiling import profile, profile2df class TestProfiling(ExtTestCase): @@ -51,6 +50,11 @@ def simple2(): self.assertIsInstance(df, pandas.DataFrame) self.assertEqual(df.loc[0, 'namefct'].split('-')[-1], 'simple2') self.assertNotEmpty(ps) + df = profile2df(ps, False) + self.assertIsInstance(df, list) + self.assertIsInstance(df[0], dict) + df = profile2df(ps, True) + self.assertIsInstance(df, pandas.DataFrame) def test_profile_pyinst(self): def simple(): diff --git a/src/pyquickhelper/jenkinshelper/yaml_helper.py b/src/pyquickhelper/jenkinshelper/yaml_helper.py index 594d7c9c..6dea34cf 100644 --- a/src/pyquickhelper/jenkinshelper/yaml_helper.py +++ b/src/pyquickhelper/jenkinshelper/yaml_helper.py @@ -460,7 +460,8 @@ def add_path_win(rows, interpreter, platform, root_project): "export PATH={0}:$PATH".format(venv_interpreter)) pat = '"{0}" -m virtualenv {1}' if isinstance(value, dict): - system_site_packages = value.get('system_site_packages', True) + system_site_packages = value.get( + 'system_site_packages', True) else: system_site_packages = True if system_site_packages: diff --git a/src/pyquickhelper/pycode/profiling.py b/src/pyquickhelper/pycode/profiling.py index 09a131eb..db916863 100644 --- a/src/pyquickhelper/pycode/profiling.py +++ b/src/pyquickhelper/pycode/profiling.py @@ -46,24 +46,57 @@ def add_rows(rows, d): return rows +def profile2df(ps, as_df=True): + """ + Converts profiling statistics into a Dataframe. + + :param ps: an instance of `pstats + `_ + :param as_df: returns the results as a dataframe (True) + or a list of dictionaries (False) + :return: a DataFrame + + :: + + import pstats + from pyquickhelper.pycode.profiling import profile2df + + ps = pstats.Stats('c:/temp/bench_ortmodule_nn_gpu6') + print(ps.strip_dirs().sort_stats(SortKey.TIME).print_stats()) + df = profile2df(pd) + print(df) + """ + rows = _process_pstats(ps, lambda x: x) + if not as_df: + return rows + + import pandas + df = pandas.DataFrame(rows) + df = df[['fct', 'file', 'ncalls1', 'ncalls2', 'tin', 'cum_tin', + 'tall', 'cum_tall']] + df = df.groupby(['fct', 'file'], as_index=False).sum().sort_values( + 'cum_tall', ascending=False).reset_index(drop=True) + return df.copy() + + def profile(fct, sort='cumulative', rootrem=None, as_df=False, pyinst_format=None, return_results=False, **kwargs): """ Profiles the execution of a function. - @param fct function to profile - @param sort see `sort_stats `_ - @param rootrem root to remove in filenames - @param as_df return the results as a dataframe and not text - @param pyinst_format format for :epkg:`pyinstrument`, if not empty, - the function uses this module or raises an exception if not - installed, the options are *text*, *textu* (text with colors), - *json*, *html* - @param return_results if True, return results as well - (in the first position) - @param kwargs additional parameters used to create the profiler - @return raw results, statistics text dump (or dataframe is *as_df* is True) + :param fct: function to profile + :param sort: see `sort_stats `_ + :param rootrem: root to remove in filenames + :param as_df: return the results as a dataframe and not text + :param pyinst_format: format for :epkg:`pyinstrument`, if not empty, + the function uses this module or raises an exception if not + installed, the options are *text*, *textu* (text with colors), + *json*, *html* + :param return_results: if True, return results as well + (in the first position) + :param kwargs: additional parameters used to create the profiler + :return: raw results, statistics text dump (or dataframe is *as_df* is True) .. plot:: diff --git a/src/pyquickhelper/sphinxext/sphinx_md_builder.py b/src/pyquickhelper/sphinxext/sphinx_md_builder.py index e9768dae..b7b06528 100644 --- a/src/pyquickhelper/sphinxext/sphinx_md_builder.py +++ b/src/pyquickhelper/sphinxext/sphinx_md_builder.py @@ -979,7 +979,7 @@ def depart_todo_node(self, node): def unknown_visit(self, node): logger = logging.getLogger("MdBuilder") logger.warning("[md] unknown visit node: '{0}' - '{1}'".format( - node.__class__.__name__, node)) + node.__class__.__name__, node)) class MdBuilder(Builder): diff --git a/src/pyquickhelper/sphinxext/sphinx_rst_builder.py b/src/pyquickhelper/sphinxext/sphinx_rst_builder.py index e14da079..6003dce0 100644 --- a/src/pyquickhelper/sphinxext/sphinx_rst_builder.py +++ b/src/pyquickhelper/sphinxext/sphinx_rst_builder.py @@ -1099,7 +1099,7 @@ def unknown_visit(self, node): return logger = logging.getLogger("RstBuilder") logger.warning("[rst] unknown visit node: '{0}' - '{1}'".format( - node.__class__.__name__, node)) + node.__class__.__name__, node)) def unknown_departure(self, node): classname = node.__class__.__name__ @@ -1112,7 +1112,7 @@ def unknown_departure(self, node): return logger = logging.getLogger("RstBuilder") logger.warning("[rst] unknown depart node: '{0}' - '{1}'".format( - node.__class__.__name__, node)) + node.__class__.__name__, node)) class _BodyPlaceholder: